From 39006fb0ffbbf658a6d28e5c747cda64dc7520dd Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 23 Sep 2018 14:24:48 -0400 Subject: [PATCH 1/7] bpo-34776: Fix dataclasses to support __future__ "annotations" mode --- Lib/dataclasses.py | 26 +++++++++++++------ Lib/test/dataclass_textanno.py | 12 +++++++++ Lib/test/test_dataclasses.py | 12 +++++++++ .../2018-09-23-14-24-37.bpo-34776.1SrQe3.rst | 1 + 4 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 Lib/test/dataclass_textanno.py create mode 100644 Misc/NEWS.d/next/Library/2018-09-23-14-24-37.bpo-34776.1SrQe3.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 28e9f75127b19b..ee97c452ea1af3 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -354,13 +354,17 @@ def _create_fn(name, args, body, *, globals=None, locals=None, locals['_return_type'] = return_type return_annotation = '->_return_type' args = ','.join(args) - body = '\n'.join(f' {b}' for b in body) + body = '\n'.join(f' {b}' for b in body) # Compute the text of the entire function. - txt = f'def {name}({args}){return_annotation}:\n{body}' + txt = f' def {name}({args}){return_annotation}:\n{body}' + + local_vars = ', '.join(locals.keys()) + txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" exec(txt, globals, locals) - return locals[name] + create_fn = locals.pop('__create_fn__') + return create_fn(**locals) def _field_assign(frozen, name, value, self_name): @@ -448,7 +452,7 @@ def _init_param(f): return f'{f.name}:_type_{f.name}{default}' -def _init_fn(fields, frozen, has_post_init, self_name): +def _init_fn(fields, frozen, has_post_init, self_name, modname): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -466,12 +470,18 @@ def _init_fn(fields, frozen, has_post_init, self_name): raise TypeError(f'non-default argument {f.name!r} ' 'follows default argument') - globals = {'MISSING': MISSING, - '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY} + globals = sys.modules[modname].__dict__ + + locals = {f'_type_{f.name}': f.type for f in fields} + locals.update({ + 'MISSING': MISSING, + '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY, + '__builtins__': builtins, + }) body_lines = [] for f in fields: - line = _field_init(f, frozen, globals, self_name) + line = _field_init(f, frozen, locals, self_name) # line is None means that this field doesn't require # initialization (it's a pseudo-field). Just skip it. if line: @@ -487,7 +497,6 @@ def _init_fn(fields, frozen, has_post_init, self_name): if not body_lines: body_lines = ['pass'] - locals = {f'_type_{f.name}': f.type for f in fields} return _create_fn('__init__', [self_name] + [_init_param(f) for f in fields if f.init], body_lines, @@ -877,6 +886,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # if possible. '__dataclass_self__' if 'self' in fields else 'self', + cls.__module__ )) # Get the fields as a list, and include only real fields. This is diff --git a/Lib/test/dataclass_textanno.py b/Lib/test/dataclass_textanno.py new file mode 100644 index 00000000000000..3eb6c943d4c434 --- /dev/null +++ b/Lib/test/dataclass_textanno.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import dataclasses + + +class Foo: + pass + + +@dataclasses.dataclass +class Bar: + foo: Foo diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 6efe785bc3282c..d08bd685153a0f 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -10,6 +10,7 @@ import unittest from unittest.mock import Mock from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional +from typing import get_type_hints from collections import deque, OrderedDict, namedtuple from functools import total_ordering @@ -2882,6 +2883,17 @@ def test_classvar_module_level_import(self): # won't exist on the instance. self.assertNotIn('not_iv4', c.__dict__) + def test_text_annotations(self): + from test import dataclass_textanno + + self.assertEqual( + get_type_hints(dataclass_textanno.Bar), + {'foo': dataclass_textanno.Foo}) + self.assertEqual( + get_type_hints(dataclass_textanno.Bar.__init__), + {'foo': dataclass_textanno.Foo, + 'return': type(None)}) + class TestMakeDataclass(unittest.TestCase): def test_simple(self): diff --git a/Misc/NEWS.d/next/Library/2018-09-23-14-24-37.bpo-34776.1SrQe3.rst b/Misc/NEWS.d/next/Library/2018-09-23-14-24-37.bpo-34776.1SrQe3.rst new file mode 100644 index 00000000000000..48d8e59453505e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-09-23-14-24-37.bpo-34776.1SrQe3.rst @@ -0,0 +1 @@ +Fix dataclasses to support __future__ "annotations" mode From ebcbf692b6c49297fe8b019a2c7a4804bbfe57ae Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 23 Sep 2018 18:06:28 -0400 Subject: [PATCH 2/7] Implement Guido's suggestion to always set correct __globals__ --- Lib/dataclasses.py | 64 ++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index ee97c452ea1af3..3c30ea0ea6af33 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -347,8 +347,8 @@ def _create_fn(name, args, body, *, globals=None, locals=None, # __builtins__ may be the "builtins" module or # the value of its "__dict__", # so make sure "__builtins__" is the module. - if globals is not None and '__builtins__' not in globals: - globals['__builtins__'] = builtins + if '__builtins__' not in locals: + locals['__builtins__'] = builtins return_annotation = '' if return_type is not MISSING: locals['_return_type'] = return_type @@ -362,9 +362,9 @@ def _create_fn(name, args, body, *, globals=None, locals=None, local_vars = ', '.join(locals.keys()) txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" - exec(txt, globals, locals) - create_fn = locals.pop('__create_fn__') - return create_fn(**locals) + ns = {} + exec(txt, globals, ns) + return ns['__create_fn__'](**locals) def _field_assign(frozen, name, value, self_name): @@ -452,7 +452,7 @@ def _init_param(f): return f'{f.name}:_type_{f.name}{default}' -def _init_fn(fields, frozen, has_post_init, self_name, modname): +def _init_fn(fields, frozen, has_post_init, self_name, globals): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -470,8 +470,6 @@ def _init_fn(fields, frozen, has_post_init, self_name, modname): raise TypeError(f'non-default argument {f.name!r} ' 'follows default argument') - globals = sys.modules[modname].__dict__ - locals = {f'_type_{f.name}': f.type for f in fields} locals.update({ 'MISSING': MISSING, @@ -505,19 +503,18 @@ def _init_fn(fields, frozen, has_post_init, self_name, modname): return_type=None) -def _repr_fn(fields): +def _repr_fn(fields, globals): return _create_fn('__repr__', ('self',), ['return self.__class__.__qualname__ + f"(' + ', '.join([f"{f.name}={{self.{f.name}!r}}" for f in fields]) + - ')"']) + ')"'], + globals=globals) -def _frozen_get_del_attr(cls, fields): - # XXX: globals is modified on the first call to _create_fn, then - # the modified version is used in the second call. Is this okay? - globals = {'cls': cls, +def _frozen_get_del_attr(cls, fields, globals): + locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} if fields: fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' @@ -529,17 +526,19 @@ def _frozen_get_del_attr(cls, fields): (f'if type(self) is cls or name in {fields_str}:', ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', f'super(cls, self).__setattr__(name, value)'), + locals=locals, globals=globals), _create_fn('__delattr__', ('self', 'name'), (f'if type(self) is cls or name in {fields_str}:', ' raise FrozenInstanceError(f"cannot delete field {name!r}")', f'super(cls, self).__delattr__(name)'), + locals=locals, globals=globals), ) -def _cmp_fn(name, op, self_tuple, other_tuple): +def _cmp_fn(name, op, self_tuple, other_tuple, globals): # Create a comparison function. If the fields in the object are # named 'x' and 'y', then self_tuple is the string # '(self.x,self.y)' and other_tuple is the string @@ -549,14 +548,16 @@ def _cmp_fn(name, op, self_tuple, other_tuple): ('self', 'other'), [ 'if other.__class__ is self.__class__:', f' return {self_tuple}{op}{other_tuple}', - 'return NotImplemented']) + 'return NotImplemented'], + globals=globals) -def _hash_fn(fields): +def _hash_fn(fields, globals): self_tuple = _tuple_str('self', fields) return _create_fn('__hash__', ('self',), - [f'return hash({self_tuple})']) + [f'return hash({self_tuple})'], + globals=globals) def _is_classvar(a_type, typing): @@ -728,14 +729,14 @@ def _set_new_attribute(cls, name, value): # take. The common case is to do nothing, so instead of providing a # function that is a no-op, use None to signify that. -def _hash_set_none(cls, fields): +def _hash_set_none(cls, fields, globals): return None -def _hash_add(cls, fields): +def _hash_add(cls, fields, globals): flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] - return _hash_fn(flds) + return _hash_fn(flds, globals) -def _hash_exception(cls, fields): +def _hash_exception(cls, fields, globals): # Raise an exception. raise TypeError(f'Cannot overwrite attribute __hash__ ' f'in class {cls.__name__}') @@ -777,6 +778,11 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # is defined by the base class, which is found first. fields = {} + if cls.__module__ in sys.modules: + globals = sys.modules[cls.__module__].__dict__ + else: + globals = {} + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, unsafe_hash, frozen)) @@ -886,7 +892,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # if possible. '__dataclass_self__' if 'self' in fields else 'self', - cls.__module__ + globals )) # Get the fields as a list, and include only real fields. This is @@ -895,7 +901,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if repr: flds = [f for f in field_list if f.repr] - _set_new_attribute(cls, '__repr__', _repr_fn(flds)) + _set_new_attribute(cls, '__repr__', _repr_fn(flds, globals)) if eq: # Create _eq__ method. There's no need for a __ne__ method, @@ -905,7 +911,8 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): other_tuple = _tuple_str('other', flds) _set_new_attribute(cls, '__eq__', _cmp_fn('__eq__', '==', - self_tuple, other_tuple)) + self_tuple, other_tuple, + globals=globals)) if order: # Create and set the ordering methods. @@ -918,13 +925,14 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): ('__ge__', '>='), ]: if _set_new_attribute(cls, name, - _cmp_fn(name, op, self_tuple, other_tuple)): + _cmp_fn(name, op, self_tuple, other_tuple, + globals=globals)): raise TypeError(f'Cannot overwrite attribute {name} ' f'in class {cls.__name__}. Consider using ' 'functools.total_ordering') if frozen: - for fn in _frozen_get_del_attr(cls, field_list): + for fn in _frozen_get_del_attr(cls, field_list, globals): if _set_new_attribute(cls, fn.__name__, fn): raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' f'in class {cls.__name__}') @@ -937,7 +945,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if hash_action: # No need to call _set_new_attribute here, since by the time # we're here the overwriting is unconditional. - cls.__hash__ = hash_action(cls, field_list) + cls.__hash__ = hash_action(cls, field_list, globals) if not getattr(cls, '__doc__'): # Create a class doc-string. From 32427a507cd7c9b5e4a846ceeacd29c35bfa4b6d Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 23 Sep 2018 18:10:27 -0400 Subject: [PATCH 3/7] Add a comment --- Lib/dataclasses.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3c30ea0ea6af33..87b639db4f8efa 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -781,6 +781,11 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if cls.__module__ in sys.modules: globals = sys.modules[cls.__module__].__dict__ else: + # Theoretically this can happen if someone writes + # a custom string to cls.__module__. In which case + # such dataclass won't be fully introspectable + # (w.r.t. typing.get_type_hints) but will still function + # correctly. globals = {} setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, From 97df9b2402ad314998f2537fc84d87ac8f75b9cc Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 23 Sep 2018 18:11:39 -0400 Subject: [PATCH 4/7] nit fix --- Lib/dataclasses.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 87b639db4f8efa..a3dd49579fe630 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -474,7 +474,6 @@ def _init_fn(fields, frozen, has_post_init, self_name, globals): locals.update({ 'MISSING': MISSING, '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY, - '__builtins__': builtins, }) body_lines = [] From 563dda4cf2e2a17cf9887e579a37ffc7064a7038 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 23 Sep 2018 21:03:38 -0400 Subject: [PATCH 5/7] rename __builtins__ for clarity --- Lib/dataclasses.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index a3dd49579fe630..c07f6094512115 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -344,11 +344,8 @@ def _create_fn(name, args, body, *, globals=None, locals=None, # worries about external callers. if locals is None: locals = {} - # __builtins__ may be the "builtins" module or - # the value of its "__dict__", - # so make sure "__builtins__" is the module. - if '__builtins__' not in locals: - locals['__builtins__'] = builtins + if 'BUILTINS' not in locals: + locals['BUILTINS'] = builtins return_annotation = '' if return_type is not MISSING: locals['_return_type'] = return_type @@ -375,7 +372,7 @@ def _field_assign(frozen, name, value, self_name): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'__builtins__.object.__setattr__({self_name},{name!r},{value})' + return f'BUILTINS.object.__setattr__({self_name},{name!r},{value})' return f'{self_name}.{name}={value}' From 8400415ddb7af585cedefb2d4688e17b89f3b117 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sun, 23 Sep 2018 22:04:28 -0400 Subject: [PATCH 6/7] Add a trailing comma --- Lib/dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index c07f6094512115..f696eb08ef68cf 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -893,7 +893,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # if possible. '__dataclass_self__' if 'self' in fields else 'self', - globals + globals, )) # Get the fields as a list, and include only real fields. This is From 71ab40568d1818d9d4968d3c3a37f6fcace6133b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 9 Dec 2019 15:30:54 +0100 Subject: [PATCH 7/7] Update 2018-09-23-14-24-37.bpo-34776.1SrQe3.rst --- .../next/Library/2018-09-23-14-24-37.bpo-34776.1SrQe3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-09-23-14-24-37.bpo-34776.1SrQe3.rst b/Misc/NEWS.d/next/Library/2018-09-23-14-24-37.bpo-34776.1SrQe3.rst index 48d8e59453505e..815a4876e0b4aa 100644 --- a/Misc/NEWS.d/next/Library/2018-09-23-14-24-37.bpo-34776.1SrQe3.rst +++ b/Misc/NEWS.d/next/Library/2018-09-23-14-24-37.bpo-34776.1SrQe3.rst @@ -1 +1 @@ -Fix dataclasses to support __future__ "annotations" mode +Fix dataclasses to support forward references in type annotations