From c2d85b7b03e472126e0561466c38adf231d0b99b Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 25 Jul 2024 10:17:32 +0300 Subject: [PATCH 1/9] gh-82128: Provide `__annotate__` method for dataclasses from `make_dataclass` --- Lib/dataclasses.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 4cba606dd8dd4d..d524adc1ad1fc8 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1527,10 +1527,11 @@ class C(Base): seen = set() annotations = {} defaults = {} + any_marker = object() for item in fields: if isinstance(item, str): name = item - tp = 'typing.Any' + tp = any_marker elif len(item) == 2: name, tp, = item elif len(item) == 3: @@ -1549,11 +1550,20 @@ class C(Base): seen.add(name) annotations[name] = tp + def annotate_method(format): + if format != 1: + raise NotImplementedError + from typing import Any + return { + ann: Any if t is any_marker else t + for ann, t in annotations.items() + } + # Update 'ns' with the user-supplied namespace plus our calculated values. def exec_body_callback(ns): + ns['__annotate__'] = annotate_method ns.update(namespace) ns.update(defaults) - ns['__annotations__'] = annotations # We use `types.new_class()` instead of simply `type()` to allow dynamic creation # of generic dataclasses. From 642f8498845592bf8e7197f1aedfd0c894bbe589 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 18 Sep 2024 09:18:03 +0300 Subject: [PATCH 2/9] Fix NameError --- Lib/dataclasses.py | 10 ++++--- Lib/test/test_dataclasses/__init__.py | 28 +++++++++++++++---- ...4-09-18-09-15-40.gh-issue-82129.GQwt3u.rst | 3 ++ 3 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index af26991ddfa399..8c8173a73f68cf 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1551,13 +1551,15 @@ class C(Base): annotations[name] = tp def annotate_method(format): - if format != 1: - raise NotImplementedError - from typing import Any - return { + from typing import Any, _convert_to_source + ann_dict = { ann: Any if t is any_marker else t for ann, t in annotations.items() } + if format == 1 or format == 2: + return ann_dict + else: + return _convert_to_source(ann_dict) # Update 'ns' with the user-supplied namespace plus our calculated values. def exec_body_callback(ns): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 6934e88d9d338c..09ad211d22d5a1 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -4159,16 +4159,34 @@ def test_no_types(self): C = make_dataclass('Point', ['x', 'y', 'z']) c = C(1, 2, 3) self.assertEqual(vars(c), {'x': 1, 'y': 2, 'z': 3}) - self.assertEqual(C.__annotations__, {'x': 'typing.Any', - 'y': 'typing.Any', - 'z': 'typing.Any'}) + self.assertEqual(C.__annotations__, {'x': typing.Any, + 'y': typing.Any, + 'z': typing.Any}) C = make_dataclass('Point', ['x', ('y', int), 'z']) c = C(1, 2, 3) self.assertEqual(vars(c), {'x': 1, 'y': 2, 'z': 3}) - self.assertEqual(C.__annotations__, {'x': 'typing.Any', + self.assertEqual(C.__annotations__, {'x': typing.Any, 'y': int, - 'z': 'typing.Any'}) + 'z': typing.Any}) + + def test_no_types_get_annotations(self): + from annotationlib import Format, get_annotations + + C = make_dataclass('C', ['x', ('y', int), 'z']) + + self.assertEqual( + get_annotations(C, format=Format.VALUE), + {'x': typing.Any, 'y': int, 'z': typing.Any}, + ) + self.assertEqual( + get_annotations(C, format=Format.FORWARDREF), + {'x': typing.Any, 'y': int, 'z': typing.Any}, + ) + self.assertEqual( + get_annotations(C, format=Format.SOURCE), + {'x': 'typing.Any', 'y': 'int', 'z': 'typing.Any'}, + ) def test_module_attr(self): self.assertEqual(ByMakeDataClass.__module__, __name__) diff --git a/Misc/NEWS.d/next/Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst b/Misc/NEWS.d/next/Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst new file mode 100644 index 00000000000000..a826e32da1c2e9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst @@ -0,0 +1,3 @@ +Fix :exc:`NameError` happenning on :func:`dataclasses.dataclass` created by +:func:`dataclasses.make_dataclass` with un-annotated fields when +:func:`typing.get_type_hints` was called on it. From 5ec3a088e2429479438c272f259943eeaf9c2ec9 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 20 Sep 2024 10:21:39 +0300 Subject: [PATCH 3/9] Address review --- Lib/dataclasses.py | 13 +++++++++--- Lib/test/test_dataclasses/__init__.py | 29 +++++++++++++++++++++------ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8c8173a73f68cf..5c4698e8a23d29 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1551,15 +1551,22 @@ class C(Base): annotations[name] = tp def annotate_method(format): + typing = sys.modules.get("typing") + if typing is None and format == annotationlib.Format.FORWARDREF: + typing_any = annotationlib.ForwardRef("Any", module="typing") + return { + ann: typing_any if t is any_marker else t + for ann, t in annotations.items() + } + from typing import Any, _convert_to_source ann_dict = { ann: Any if t is any_marker else t for ann, t in annotations.items() } - if format == 1 or format == 2: - return ann_dict - else: + if format == annotationlib.Format.SOURCE: return _convert_to_source(ann_dict) + return ann_dict # Update 'ns' with the user-supplied namespace plus our calculated values. def exec_body_callback(ns): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 09ad211d22d5a1..7e988731f415c3 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -5,6 +5,7 @@ from dataclasses import * import abc +import annotationlib import io import pickle import inspect @@ -23,6 +24,7 @@ import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation. from test import support +from test.support import import_helper # Just any custom exception we can catch. class CustomError(Exception): pass @@ -3667,7 +3669,6 @@ class A(WithDictSlot): ... @support.cpython_only def test_dataclass_slot_dict_ctype(self): # https://github.com/python/cpython/issues/123935 - from test.support import import_helper # Skips test if `_testcapi` is not present: _testcapi = import_helper.import_module('_testcapi') @@ -4171,23 +4172,39 @@ def test_no_types(self): 'z': typing.Any}) def test_no_types_get_annotations(self): - from annotationlib import Format, get_annotations - C = make_dataclass('C', ['x', ('y', int), 'z']) self.assertEqual( - get_annotations(C, format=Format.VALUE), + annotationlib.get_annotations(C, format=annotationlib.Format.VALUE), {'x': typing.Any, 'y': int, 'z': typing.Any}, ) self.assertEqual( - get_annotations(C, format=Format.FORWARDREF), + annotationlib.get_annotations( + C, format=annotationlib.Format.FORWARDREF), {'x': typing.Any, 'y': int, 'z': typing.Any}, ) self.assertEqual( - get_annotations(C, format=Format.SOURCE), + annotationlib.get_annotations( + C, format=annotationlib.Format.SOURCE), {'x': 'typing.Any', 'y': 'int', 'z': 'typing.Any'}, ) + def test_no_types_no_typing_import(self): + import sys + + C = make_dataclass('C', ['x', ('y', int)]) + + with import_helper.CleanImport('typing'): + self.assertEqual( + annotationlib.get_annotations( + C, format=annotationlib.Format.FORWARDREF), + { + 'x': annotationlib.ForwardRef('Any', module='typing'), + 'y': int, + }, + ) + self.assertNotIn('typing', sys.modules) + def test_module_attr(self): self.assertEqual(ByMakeDataClass.__module__, __name__) self.assertEqual(ByMakeDataClass(1).__module__, __name__) From 43caa3fadf83428707b4abded312ceb238c52388 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 24 Sep 2024 09:19:28 +0300 Subject: [PATCH 4/9] Update Misc/NEWS.d/next/Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst Co-authored-by: Carl Meyer --- .../Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst b/Misc/NEWS.d/next/Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst index a826e32da1c2e9..b8661f9eeaa5c9 100644 --- a/Misc/NEWS.d/next/Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst +++ b/Misc/NEWS.d/next/Library/2024-09-18-09-15-40.gh-issue-82129.GQwt3u.rst @@ -1,3 +1,2 @@ -Fix :exc:`NameError` happenning on :func:`dataclasses.dataclass` created by -:func:`dataclasses.make_dataclass` with un-annotated fields when -:func:`typing.get_type_hints` was called on it. +Fix :exc:`NameError` when calling :func:`typing.get_type_hints` on a :func:`dataclasses.dataclass` created by +:func:`dataclasses.make_dataclass` with un-annotated fields. From 33b279dfcfca200bf8e862b14fffe9997214e371 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 25 Sep 2024 14:13:13 +0300 Subject: [PATCH 5/9] Introduce `_ANY_MARKER` --- Lib/dataclasses.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 5c4698e8a23d29..23cb1a22ebfbbd 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -244,6 +244,10 @@ def __repr__(self): property, }) +# Any marker is used in `make_dataclass` to mark unannotated fields as `Any` +# without importing `typing` module. +_ANY_MARKER = object() + class InitVar: __slots__ = ('type', ) @@ -1527,11 +1531,10 @@ class C(Base): seen = set() annotations = {} defaults = {} - any_marker = object() for item in fields: if isinstance(item, str): name = item - tp = any_marker + tp = _ANY_MARKER elif len(item) == 2: name, tp, = item elif len(item) == 3: @@ -1555,13 +1558,13 @@ def annotate_method(format): if typing is None and format == annotationlib.Format.FORWARDREF: typing_any = annotationlib.ForwardRef("Any", module="typing") return { - ann: typing_any if t is any_marker else t + ann: typing_any if t is _ANY_MARKER else t for ann, t in annotations.items() } from typing import Any, _convert_to_source ann_dict = { - ann: Any if t is any_marker else t + ann: Any if t is _ANY_MARKER else t for ann, t in annotations.items() } if format == annotationlib.Format.SOURCE: From 3b26e36e55f85a73866fa0ae04cb190ea6be01de Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 15 Oct 2024 12:47:26 +0300 Subject: [PATCH 6/9] WIP --- Lib/dataclasses.py | 6 +++--- Lib/test/test_dataclasses/__init__.py | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 9bc13a67904086..515181708c3c27 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1620,13 +1620,13 @@ def annotate_method(format): for ann, t in annotations.items() } - from typing import Any, _convert_to_source + from typing import Any ann_dict = { ann: Any if t is _ANY_MARKER else t for ann, t in annotations.items() } - if format == annotationlib.Format.SOURCE: - return _convert_to_source(ann_dict) + if format == annotationlib.Format.STRING: + return annotationlib.annotations_to_string(ann_dict) return ann_dict # Update 'ns' with the user-supplied namespace plus our calculated values. diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 0b65c5d86d6ae3..4574ce20693d4c 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -13,6 +13,7 @@ import types import weakref import traceback +import sys import unittest from unittest.mock import Mock from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol, DefaultDict @@ -4247,16 +4248,16 @@ def test_no_types_get_annotations(self): ) self.assertEqual( annotationlib.get_annotations( - C, format=annotationlib.Format.SOURCE), + C, format=annotationlib.Format.STRING), {'x': 'typing.Any', 'y': 'int', 'z': 'typing.Any'}, ) def test_no_types_no_typing_import(self): - import sys - - C = make_dataclass('C', ['x', ('y', int)]) - with import_helper.CleanImport('typing'): + self.assertNotIn('typing', sys.modules) + C = make_dataclass('C', ['x', ('y', int)]) + + self.assertNotIn('typing', sys.modules) self.assertEqual( annotationlib.get_annotations( C, format=annotationlib.Format.FORWARDREF), From 309da4c011a46d6d56220c3a3ff8d88177c43f66 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 4 May 2025 14:34:50 -0700 Subject: [PATCH 7/9] alt --- Lib/dataclasses.py | 33 ++++++++++++++++----------- Lib/test/test_dataclasses/__init__.py | 16 ------------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 8fde48e99ab313..b4d549f7156573 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1615,22 +1615,29 @@ class C(Base): annotations[name] = tp def annotate_method(format): - typing = sys.modules.get("typing") - if typing is None and format == annotationlib.Format.FORWARDREF: - typing_any = annotationlib.ForwardRef("Any", module="typing") - return { - ann: typing_any if t is _ANY_MARKER else t - for ann, t in annotations.items() - } - - from typing import Any - ann_dict = { - ann: Any if t is _ANY_MARKER else t + def get_any(): + match format: + case annotationlib.Format.STRING: + return 'typing.Any' + case annotationlib.Format.FORWARDREF: + typing = sys.modules.get("typing") + if typing is None: + return annotationlib.ForwardRef("Any", module="typing") + else: + return typing.Any + case annotationlib.Format.VALUE: + from typing import Any + return Any + case _: + raise NotImplementedError + annos = { + ann: get_any() if t is _ANY_MARKER else t for ann, t in annotations.items() } if format == annotationlib.Format.STRING: - return annotationlib.annotations_to_string(ann_dict) - return ann_dict + return annotationlib.annotations_to_string(annos) + else: + return annos # Update 'ns' with the user-supplied namespace plus our calculated values. def exec_body_callback(ns): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 060dbcc9303cc6..f6a1ed475dae02 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -4277,22 +4277,6 @@ def test_no_types_get_annotations(self): {'x': 'typing.Any', 'y': 'int', 'z': 'typing.Any'}, ) - def test_no_types_no_typing_import(self): - with import_helper.CleanImport('typing'): - self.assertNotIn('typing', sys.modules) - C = make_dataclass('C', ['x', ('y', int)]) - - self.assertNotIn('typing', sys.modules) - self.assertEqual( - annotationlib.get_annotations( - C, format=annotationlib.Format.FORWARDREF), - { - 'x': annotationlib.ForwardRef('Any', module='typing'), - 'y': int, - }, - ) - self.assertNotIn('typing', sys.modules) - def test_module_attr(self): self.assertEqual(ByMakeDataClass.__module__, __name__) self.assertEqual(ByMakeDataClass(1).__module__, __name__) From 3d7eb6edb8438414dd6e683377ec57ad9edb7fe6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 4 May 2025 15:17:03 -0700 Subject: [PATCH 8/9] better --- Lib/annotationlib.py | 2 +- Lib/dataclasses.py | 17 ++++++++++++----- Lib/test/test_dataclasses/__init__.py | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index cd24679f30abee..c4bbda4f06fdd6 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -788,7 +788,7 @@ def get_annotations( # For FORWARDREF, we use __annotations__ if it exists try: ann = _get_dunder_annotations(obj) - except NameError: + except Exception: pass else: if ann is not None: diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index b4d549f7156573..b1ea59ba3d16cd 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1614,6 +1614,8 @@ class C(Base): seen.add(name) annotations[name] = tp + value_blocked = True + def annotate_method(format): def get_any(): match format: @@ -1626,6 +1628,8 @@ def get_any(): else: return typing.Any case annotationlib.Format.VALUE: + if value_blocked: + raise NotImplementedError from typing import Any return Any case _: @@ -1641,13 +1645,14 @@ def get_any(): # Update 'ns' with the user-supplied namespace plus our calculated values. def exec_body_callback(ns): - ns['__annotate__'] = annotate_method ns.update(namespace) ns.update(defaults) # We use `types.new_class()` instead of simply `type()` to allow dynamic creation # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) + # For now, set annotations including the _ANY_MARKER. + cls.__annotate__ = annotate_method # For pickling to work, the __module__ variable needs to be set to the frame # where the dataclass is created. @@ -1663,10 +1668,12 @@ def exec_body_callback(ns): cls.__module__ = module # Apply the normal provided decorator. - return decorator(cls, init=init, repr=repr, eq=eq, order=order, - unsafe_hash=unsafe_hash, frozen=frozen, - match_args=match_args, kw_only=kw_only, slots=slots, - weakref_slot=weakref_slot) + cls = decorator(cls, init=init, repr=repr, eq=eq, order=order, + unsafe_hash=unsafe_hash, frozen=frozen, + match_args=match_args, kw_only=kw_only, slots=slots, + weakref_slot=weakref_slot) + value_blocked = False + return cls def replace(obj, /, **changes): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index f6a1ed475dae02..ac78f8327b808e 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -4277,6 +4277,28 @@ def test_no_types_get_annotations(self): {'x': 'typing.Any', 'y': 'int', 'z': 'typing.Any'}, ) + def test_no_types_no_typing_import(self): + with import_helper.CleanImport('typing'): + self.assertNotIn('typing', sys.modules) + C = make_dataclass('C', ['x', ('y', int)]) + + self.assertNotIn('typing', sys.modules) + self.assertEqual( + C.__annotate__(annotationlib.Format.FORWARDREF), + { + 'x': annotationlib.ForwardRef('Any', module='typing'), + 'y': int, + }, + ) + self.assertNotIn('typing', sys.modules) + + for field in fields(C): + if field.name == "x": + self.assertEqual(field.type, annotationlib.ForwardRef('Any', module='typing')) + else: + self.assertEqual(field.name, "y") + self.assertIs(field.type, int) + def test_module_attr(self): self.assertEqual(ByMakeDataClass.__module__, __name__) self.assertEqual(ByMakeDataClass(1).__module__, __name__) From 95650892fa17fe9edc7497d0e0c2ac001638c9ed Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 4 May 2025 15:18:45 -0700 Subject: [PATCH 9/9] comment --- Lib/dataclasses.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index b1ea59ba3d16cd..86d29df0639184 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1614,6 +1614,10 @@ class C(Base): seen.add(name) annotations[name] = tp + # We initially block the VALUE format, because inside dataclass() we'll + # call get_annotations(), which will try the VALUE format first. If we don't + # block, that means we'd always end up eagerly importing typing here, which + # is what we're trying to avoid. value_blocked = True def annotate_method(format): @@ -1672,6 +1676,7 @@ def exec_body_callback(ns): unsafe_hash=unsafe_hash, frozen=frozen, match_args=match_args, kw_only=kw_only, slots=slots, weakref_slot=weakref_slot) + # Now that the class is ready, allow the VALUE format. value_blocked = False return cls