From fea7cf1466d7fef90b8c17011d296b95112e034f Mon Sep 17 00:00:00 2001 From: Vadim Pushtaev Date: Wed, 25 Jul 2018 00:18:47 +0300 Subject: [PATCH 1/5] bpo-34213: frozen dataclass with "object" attr bug --- Lib/dataclasses.py | 33 ++++++++++++++++++++++----------- Lib/test/test_dataclasses.py | 14 ++++++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index e00a125bbd8711..51ac724c2d4459 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -186,6 +186,10 @@ def __repr__(self): # @dataclass. _PARAMS = '__dataclass_params__' +# The name of an attribute on the class that stores a reference to +# the object class in case it is shadowed by field name. +_OBJECT = '__dataclass_object__' + # The name of the function, that if it exists, is called at the end of # __init__. _POST_INIT_NAME = '__post_init__' @@ -357,7 +361,7 @@ def _create_fn(name, args, body, *, globals=None, locals=None, return locals[name] -def _field_assign(frozen, name, value, self_name): +def _field_assign(frozen, name, value, self_name, object_expression): # If we're a frozen class, then assign to our fields in __init__ # via object.__setattr__. Otherwise, just use a simple # assignment. @@ -365,11 +369,11 @@ def _field_assign(frozen, name, value, self_name): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'object.__setattr__({self_name},{name!r},{value})' + return f'{object_expression}.__setattr__({self_name},{name!r},{value})' return f'{self_name}.{name}={value}' -def _field_init(f, frozen, globals, self_name): +def _field_init(f, frozen, globals, self_name, object_expression): # Return the text of the line in the body of __init__ that will # initialize this field. @@ -420,7 +424,7 @@ def _field_init(f, frozen, globals, self_name): return None # Now, actually generate the field assignment. - return _field_assign(frozen, f.name, value, self_name) + return _field_assign(frozen, f.name, value, self_name, object_expression) def _init_param(f): @@ -442,7 +446,7 @@ def _init_param(f): return f'{f.name}:_type_{f.name}{default}' -def _init_fn(fields, frozen, has_post_init, self_name): +def _init_fn(fields, frozen, has_post_init, self_name, object_expression): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -465,7 +469,7 @@ def _init_fn(fields, frozen, has_post_init, self_name): body_lines = [] for f in fields: - line = _field_init(f, frozen, globals, self_name) + line = _field_init(f, frozen, globals, self_name, object_expression) # line is None means that this field doesn't require # initialization (it's a pseudo-field). Just skip it. if line: @@ -841,6 +845,9 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # also marks this class as being a dataclass. setattr(cls, _FIELDS, fields) + # Copy reference to "object" in case it is shadowed by a field name. + setattr(cls, _OBJECT, object) + # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically # set __hash__ to None. This is a heuristic, as it's possible @@ -862,15 +869,19 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Include InitVars and regular fields (so, not ClassVars). flds = [f for f in fields.values() if f._field_type in (_FIELD, _FIELD_INITVAR)] + # The name to use for the "self" param in __init__. + # Use "self" if possible. + self_name = '__dataclass_self__' if 'self' in fields else 'self' + # The way to refer to "object" in __init__. + # Use "object" if possible. + object_expression = (f'{self_name}.{_OBJECT}' + if 'object' in fields else 'object') _set_new_attribute(cls, '__init__', _init_fn(flds, frozen, has_post_init, - # The name to use for the "self" - # param in __init__. Use "self" - # if possible. - '__dataclass_self__' if 'self' in fields - else 'self', + self_name, + object_expression, )) # Get the fields as a list, and include only real fields. This is diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index c5140e8d1d9c94..6ad4a9128d28ff 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -192,6 +192,20 @@ class C: first = next(iter(sig.parameters)) self.assertEqual('self', first) + def test_field_named_object(self): + @dataclass + class C: + object: str + c=C('foo') + self.assertEqual(c.object, 'foo') + + def test_field_named_object_frozen(self): + @dataclass(frozen=True) + class C: + object: str + c=C('foo') + self.assertEqual(c.object, 'foo') + def test_0_field_compare(self): # Ensure that order=False is the default. @dataclass From 3171e9c46ccff09cdea3ff0aa6b31767d04f45ef Mon Sep 17 00:00:00 2001 From: Vadim Pushtaev Date: Wed, 25 Jul 2018 00:40:39 +0300 Subject: [PATCH 2/5] bpo-34213: news for frozen dataclasses bug --- .../next/Library/2018-07-25-00-40-14.bpo-34213.O15MgP.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-07-25-00-40-14.bpo-34213.O15MgP.rst diff --git a/Misc/NEWS.d/next/Library/2018-07-25-00-40-14.bpo-34213.O15MgP.rst b/Misc/NEWS.d/next/Library/2018-07-25-00-40-14.bpo-34213.O15MgP.rst new file mode 100644 index 00000000000000..a762b59e2a13e8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-07-25-00-40-14.bpo-34213.O15MgP.rst @@ -0,0 +1,2 @@ +Frozen dataclasses didn't work properly with an attribute called "object". +Now they do. From 4d7ee274a3901c0a4b57b07ae9b3063854a6cd70 Mon Sep 17 00:00:00 2001 From: Vadim Pushtaev Date: Thu, 26 Jul 2018 15:55:42 +0300 Subject: [PATCH 3/5] bpo-34213: frozen dataclass with "object" attr bug --- Lib/dataclasses.py | 33 ++++++----------- Lib/test/test_dataclasses.py | 37 ++++++++++++++++++- .../2018-07-25-00-40-14.bpo-34213.O15MgP.rst | 3 +- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 51ac724c2d4459..3264f3b23bcba3 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -186,10 +186,6 @@ def __repr__(self): # @dataclass. _PARAMS = '__dataclass_params__' -# The name of an attribute on the class that stores a reference to -# the object class in case it is shadowed by field name. -_OBJECT = '__dataclass_object__' - # The name of the function, that if it exists, is called at the end of # __init__. _POST_INIT_NAME = '__post_init__' @@ -361,7 +357,7 @@ def _create_fn(name, args, body, *, globals=None, locals=None, return locals[name] -def _field_assign(frozen, name, value, self_name, object_expression): +def _field_assign(frozen, name, value, self_name): # If we're a frozen class, then assign to our fields in __init__ # via object.__setattr__. Otherwise, just use a simple # assignment. @@ -369,11 +365,11 @@ def _field_assign(frozen, name, value, self_name, object_expression): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'{object_expression}.__setattr__({self_name},{name!r},{value})' + return f'__builtins__["object"].__setattr__({self_name},{name!r},{value})' return f'{self_name}.{name}={value}' -def _field_init(f, frozen, globals, self_name, object_expression): +def _field_init(f, frozen, globals, self_name): # Return the text of the line in the body of __init__ that will # initialize this field. @@ -424,7 +420,7 @@ def _field_init(f, frozen, globals, self_name, object_expression): return None # Now, actually generate the field assignment. - return _field_assign(frozen, f.name, value, self_name, object_expression) + return _field_assign(frozen, f.name, value, self_name) def _init_param(f): @@ -446,7 +442,7 @@ def _init_param(f): return f'{f.name}:_type_{f.name}{default}' -def _init_fn(fields, frozen, has_post_init, self_name, object_expression): +def _init_fn(fields, frozen, has_post_init, self_name): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -469,7 +465,7 @@ def _init_fn(fields, frozen, has_post_init, self_name, object_expression): body_lines = [] for f in fields: - line = _field_init(f, frozen, globals, self_name, object_expression) + line = _field_init(f, frozen, globals, self_name) # line is None means that this field doesn't require # initialization (it's a pseudo-field). Just skip it. if line: @@ -845,9 +841,6 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # also marks this class as being a dataclass. setattr(cls, _FIELDS, fields) - # Copy reference to "object" in case it is shadowed by a field name. - setattr(cls, _OBJECT, object) - # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically # set __hash__ to None. This is a heuristic, as it's possible @@ -869,19 +862,15 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Include InitVars and regular fields (so, not ClassVars). flds = [f for f in fields.values() if f._field_type in (_FIELD, _FIELD_INITVAR)] - # The name to use for the "self" param in __init__. - # Use "self" if possible. - self_name = '__dataclass_self__' if 'self' in fields else 'self' - # The way to refer to "object" in __init__. - # Use "object" if possible. - object_expression = (f'{self_name}.{_OBJECT}' - if 'object' in fields else 'object') _set_new_attribute(cls, '__init__', _init_fn(flds, frozen, has_post_init, - self_name, - object_expression, + # The name to use for the "self" + # param in __init__. Use "self" + # if possible. + '__dataclass_self__' if 'self' in fields + else 'self', )) # Get the fields as a list, and include only real fields. This is diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 6ad4a9128d28ff..c2cc086be82051 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -196,16 +196,49 @@ def test_field_named_object(self): @dataclass class C: object: str - c=C('foo') + c = C('foo') self.assertEqual(c.object, 'foo') def test_field_named_object_frozen(self): @dataclass(frozen=True) class C: object: str - c=C('foo') + c = C('foo') self.assertEqual(c.object, 'foo') + def test_field_named_like_builtin(self): + # Attribute names can shadow built-in names + # since code generation is used. + # Ensure that this is not happening. + exclusions = {'None', 'True', 'False'} + builtins = sorted( + b for b in __builtins__.__dict__.keys() + if not b.startswith('__') and b not in exclusions + ) + C = make_dataclass('C', [(name, str) for name in builtins]) + + c = C(*[name for name in builtins]) + + for name in builtins: + self.assertEqual(getattr(c, name), name) + + def test_field_named_like_builtin_frozen(self): + # Attribute names can shadow built-in names + # since code generation is used. + # Ensure that this is not happening + # for frozen data classes. + exclusions = {'None', 'True', 'False'} + builtins = sorted( + b for b in __builtins__.__dict__.keys() + if not b.startswith('__') and b not in exclusions + ) + C = make_dataclass('C', [(name, str) for name in builtins], frozen=True) + + c = C(*[name for name in builtins]) + + for name in builtins: + self.assertEqual(getattr(c, name), name) + def test_0_field_compare(self): # Ensure that order=False is the default. @dataclass diff --git a/Misc/NEWS.d/next/Library/2018-07-25-00-40-14.bpo-34213.O15MgP.rst b/Misc/NEWS.d/next/Library/2018-07-25-00-40-14.bpo-34213.O15MgP.rst index a762b59e2a13e8..28012af457283d 100644 --- a/Misc/NEWS.d/next/Library/2018-07-25-00-40-14.bpo-34213.O15MgP.rst +++ b/Misc/NEWS.d/next/Library/2018-07-25-00-40-14.bpo-34213.O15MgP.rst @@ -1,2 +1 @@ -Frozen dataclasses didn't work properly with an attribute called "object". -Now they do. +Allow frozen dataclasses to have a field named "object". Previously this conflicted with an internal use of "object". From 4b0ea3efb09f13e69e0736b9d40c15699ffec915 Mon Sep 17 00:00:00 2001 From: Vadim Pushtaev Date: Thu, 26 Jul 2018 17:24:51 +0300 Subject: [PATCH 4/5] bpo-34213: __builtins__ bug --- Lib/dataclasses.py | 8 +++++++- Lib/test/test_dataclasses.py | 23 +++++++++++++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3264f3b23bcba3..747114f6cb4054 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -4,6 +4,7 @@ import types import inspect import keyword +import builtins __all__ = ['dataclass', 'field', @@ -343,6 +344,11 @@ def _create_fn(name, args, body, *, globals=None, locals=None, # worries about external callers. if locals is None: locals = {} + # __builtins__ may be the "builtins" module or + # the value of its "__dict__", + # so make sure "__builtins" is the module. + if globals is not None and '__builtins__' not in globals: + globals['__builtins__'] = builtins return_annotation = '' if return_type is not MISSING: locals['_return_type'] = return_type @@ -365,7 +371,7 @@ def _field_assign(frozen, name, value, self_name): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'__builtins__["object"].__setattr__({self_name},{name!r},{value})' + return f'__builtins__.object.__setattr__({self_name},{name!r},{value})' return f'{self_name}.{name}={value}' diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index c2cc086be82051..4c93513956a2d5 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -6,6 +6,7 @@ import pickle import inspect +import builtins import unittest from unittest.mock import Mock from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional @@ -211,15 +212,16 @@ def test_field_named_like_builtin(self): # since code generation is used. # Ensure that this is not happening. exclusions = {'None', 'True', 'False'} - builtins = sorted( - b for b in __builtins__.__dict__.keys() + builtins_names = sorted( + b for b in builtins.__dict__.keys() if not b.startswith('__') and b not in exclusions ) - C = make_dataclass('C', [(name, str) for name in builtins]) + attributes = [(name, str) for name in builtins_names] + C = make_dataclass('C', attributes) - c = C(*[name for name in builtins]) + c = C(*[name for name in builtins_names]) - for name in builtins: + for name in builtins_names: self.assertEqual(getattr(c, name), name) def test_field_named_like_builtin_frozen(self): @@ -228,15 +230,16 @@ def test_field_named_like_builtin_frozen(self): # Ensure that this is not happening # for frozen data classes. exclusions = {'None', 'True', 'False'} - builtins = sorted( - b for b in __builtins__.__dict__.keys() + builtins_names = sorted( + b for b in builtins.__dict__.keys() if not b.startswith('__') and b not in exclusions ) - C = make_dataclass('C', [(name, str) for name in builtins], frozen=True) + attributes = [(name, str) for name in builtins_names] + C = make_dataclass('C', attributes, frozen=True) - c = C(*[name for name in builtins]) + c = C(*[name for name in builtins_names]) - for name in builtins: + for name in builtins_names: self.assertEqual(getattr(c, name), name) def test_0_field_compare(self): From d08afd9821ecd1002ffc61aa9c06b73777982b6c Mon Sep 17 00:00:00 2001 From: Vadim Pushtaev Date: Thu, 26 Jul 2018 17:42:22 +0300 Subject: [PATCH 5/5] bpo-34213: "__builtins" typo fix --- Lib/dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 747114f6cb4054..a43d07693ac35e 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -346,7 +346,7 @@ def _create_fn(name, args, body, *, globals=None, locals=None, locals = {} # __builtins__ may be the "builtins" module or # the value of its "__dict__", - # so make sure "__builtins" is the module. + # so make sure "__builtins__" is the module. if globals is not None and '__builtins__' not in globals: globals['__builtins__'] = builtins return_annotation = ''