From 15ea9540006c7aff3ddbf0fe2b5328e70ad01b30 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 16:44:53 -0700 Subject: [PATCH 1/7] gh-125618: Make FORWARDREF format succeed more often Fixes #125618. --- Doc/library/annotationlib.rst | 16 +- Lib/annotationlib.py | 165 ++++++++++++------ Lib/test/test_annotationlib.py | 72 +++++++- ...-04-22-16-35-37.gh-issue-125618.PEocn3.rst | 3 + 4 files changed, 201 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-22-16-35-37.gh-issue-125618.PEocn3.rst diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 7946cd3a3ced34..16d7e71bbcd1b4 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -132,7 +132,7 @@ Classes Values are real annotation values (as per :attr:`Format.VALUE` format) for defined values, and :class:`ForwardRef` proxies for undefined - values. Real objects may contain references to, :class:`ForwardRef` + values. Real objects may contain references to :class:`ForwardRef` proxy objects. .. attribute:: STRING @@ -172,14 +172,22 @@ Classes :class:`~ForwardRef`. The string may not be exactly equivalent to the original source. - .. method:: evaluate(*, owner=None, globals=None, locals=None, type_params=None) + .. method:: evaluate(*, owner=None, globals=None, locals=None, type_params=None, + format=Format.VALUE) Evaluate the forward reference, returning its value. - This may throw an exception, such as :exc:`NameError`, if the forward + If the *format* argument is :attr:`~Format.VALUE` (the default), + this method may throw an exception, such as :exc:`NameError`, if the forward reference refers to a name that cannot be resolved. The arguments to this method can be used to provide bindings for names that would otherwise - be undefined. + be undefined. If the *format* argument is :attr:`~Format.FORWARDREF`, + the method will never throw an exception, but may return a :class:`~ForwardRef` + instance. For example, if the forward reference object contains the code + ``list[undefined]``, where ``undefined`` is a name that is not defined, + evaluating it with the :attr:`~Format.FORWARDREF` format will return + ``list[ForwardRef('undefined')]``. If the *format* argument is + :attr:`~Format.STRING`, the method will return :attr:`~ForwardRef.__forward_arg__`. The *owner* parameter provides the preferred mechanism for passing scope information to this method. The owner of a :class:`~ForwardRef` is the diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 971f636f9714d7..748baa495a51c1 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -90,11 +90,21 @@ def __init__( def __init_subclass__(cls, /, *args, **kwds): raise TypeError("Cannot subclass ForwardRef") - def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): + def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None, + format=Format.VALUE): """Evaluate the forward reference and return the value. If the forward reference cannot be evaluated, raise an exception. """ + match format: + case Format.STRING: + return self.__forward_arg__ + case Format.VALUE: + is_forwardref_format = False + case Format.FORWARDREF: + is_forwardref_format = True + case _: + raise NotImplementedError(format) if self.__cell__ is not None: try: return self.__cell__.cell_contents @@ -155,17 +165,33 @@ def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None): arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): if arg in locals: - value = locals[arg] + return locals[arg] elif arg in globals: - value = globals[arg] + return globals[arg] elif hasattr(builtins, arg): return getattr(builtins, arg) + elif is_forwardref_format: + return self else: raise NameError(arg) else: code = self.__forward_code__ - value = eval(code, globals=globals, locals=locals) - return value + try: + return eval(code, globals=globals, locals=locals) + except Exception: + if not is_forwardref_format: + raise + new_locals = _StringifierDict( + {**builtins.__dict__, **locals}, globals=globals, owner=owner, + is_class=self.__forward_is_class__ + ) + try: + result = eval(code, globals=globals, locals=new_locals) + except Exception: + return self + else: + new_locals.transmogrify() + return result def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): import typing @@ -478,6 +504,14 @@ def __missing__(self, key): self.stringifiers.append(fwdref) return fwdref + def transmogrify(self): + for obj in self.stringifiers: + obj.__class__ = ForwardRef + obj.__stringifier_dict__ = None # not needed for ForwardRef + if isinstance(obj.__ast_node__, str): + obj.__arg__ = obj.__ast_node__ + obj.__ast_node__ = None + def call_evaluate_function(evaluate, format, *, owner=None): """Call an evaluate function. Evaluate functions are normally generated for @@ -522,19 +556,10 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # convert each of those into a string to get an approximation of the # original source. globals = _StringifierDict({}) - if annotate.__closure__: - freevars = annotate.__code__.co_freevars - new_closure = [] - for i, cell in enumerate(annotate.__closure__): - if i < len(freevars): - name = freevars[i] - else: - name = "__cell__" - fwdref = _Stringifier(name, stringifier_dict=globals) - new_closure.append(types.CellType(fwdref)) - closure = tuple(new_closure) - else: - closure = None + is_class = isinstance(owner, type) + closure = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) func = types.FunctionType( annotate.__code__, globals, @@ -570,32 +595,30 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): namespace = {**annotate.__builtins__, **annotate.__globals__} is_class = isinstance(owner, type) globals = _StringifierDict(namespace, annotate.__globals__, owner, is_class) - if annotate.__closure__: - freevars = annotate.__code__.co_freevars - new_closure = [] - for i, cell in enumerate(annotate.__closure__): - try: - cell.cell_contents - except ValueError: - if i < len(freevars): - name = freevars[i] - else: - name = "__cell__" - fwdref = _Stringifier( - name, - cell=cell, - owner=owner, - globals=annotate.__globals__, - is_class=is_class, - stringifier_dict=globals, - ) - globals.stringifiers.append(fwdref) - new_closure.append(types.CellType(fwdref)) - else: - new_closure.append(cell) - closure = tuple(new_closure) + closure = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=True + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + try: + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except Exception: + pass else: - closure = None + globals.transmogrify() + return result + + # Try again, but do not provide any globals. This allows us to return + # a value in certain cases where an exception gets raised during evaluation. + globals = _StringifierDict({}, annotate.__globals__, owner, is_class) + closure = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) func = types.FunctionType( annotate.__code__, globals, @@ -604,13 +627,21 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): kwdefaults=annotate.__kwdefaults__, ) result = func(Format.VALUE_WITH_FAKE_GLOBALS) - for obj in globals.stringifiers: - obj.__class__ = ForwardRef - obj.__stringifier_dict__ = None # not needed for ForwardRef - if isinstance(obj.__ast_node__, str): - obj.__arg__ = obj.__ast_node__ - obj.__ast_node__ = None - return result + globals.transmogrify() + if _is_evaluate: + if isinstance(result, ForwardRef): + return result.evaluate(format=Format.FORWARDREF) + else: + return result + else: + return { + key: ( + val.evaluate(format=Format.FORWARDREF) + if isinstance(val, ForwardRef) + else val + ) + for key, val in result.items() + } elif format == Format.VALUE: # Should be impossible because __annotate__ functions must not raise # NotImplementedError for this format. @@ -619,6 +650,40 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): raise ValueError(f"Invalid format: {format!r}") +def _build_closure(annotate, owner, is_class, stringifier_dict, *, + allow_evaluation): + if not annotate.__closure__: + return None + freevars = annotate.__code__.co_freevars + new_closure = [] + for i, cell in enumerate(annotate.__closure__): + if i < len(freevars): + name = freevars[i] + else: + name = "__cell__" + new_cell = None + if allow_evaluation: + try: + cell.cell_contents + except ValueError: + pass + else: + new_cell = cell + if new_cell is None: + fwdref = _Stringifier( + name, + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + stringifier_dict=globals, + ) + stringifier_dict.stringifiers.append(fwdref) + new_cell = types.CellType(fwdref) + new_closure.append(new_cell) + return tuple(new_closure) + + def get_annotate_function(obj): """Get the __annotate__ function for an object. diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 0890be529a7e52..2c3e04df398bf9 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -307,7 +307,7 @@ def test_special_attrs(self): # Forward refs provide a different introspection API. __name__ and # __qualname__ make little sense for forward refs as they can store # complex typing expressions. - fr = annotationlib.ForwardRef("set[Any]") + fr = ForwardRef("set[Any]") self.assertFalse(hasattr(fr, "__name__")) self.assertFalse(hasattr(fr, "__qualname__")) self.assertEqual(fr.__module__, "annotationlib") @@ -317,6 +317,38 @@ def test_special_attrs(self): with self.assertRaises(TypeError): pickle.dumps(fr, proto) + def test_evaluate_string_format(self): + fr = ForwardRef("set[Any]") + self.assertEqual(fr.evaluate(format=Format.STRING), "set[Any]") + + def test_evaluate_forwardref_format(self): + fr = ForwardRef("undef") + evaluated = fr.evaluate(format=Format.FORWARDREF) + self.assertIs(fr, evaluated) + + fr = ForwardRef("set[undefined]") + evaluated = fr.evaluate(format=Format.FORWARDREF) + self.assertEqual( + evaluated, + set[support.EqualToForwardRef("undefined")], + ) + + fr = ForwardRef("a + b") + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF), + support.EqualToForwardRef("a + b"), + ) + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF, locals={"a": 1, "b": 2}), + 3, + ) + + fr = ForwardRef('"a" + 1') + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF), + support.EqualToForwardRef('"a" + 1'), + ) + def test_evaluate_with_type_params(self): class Gen[T]: alias = int @@ -1054,6 +1086,44 @@ def test_pep_695_generics_with_future_annotations_nested_in_function(self): set(results.generic_func.__type_params__), ) + maxDiff = None + + def test_partial_evaluation(self): + def f( + x: builtins.undef, + y: list[int], + z: 1 + int, + a: builtins.int, + b: [builtins.undef, builtins.int], + ): + pass + + self.assertEqual( + annotationlib.get_annotations(f, format=Format.FORWARDREF), + { + "x": support.EqualToForwardRef("builtins.undef", owner=f), + "y": list[int], + "z": support.EqualToForwardRef("1 + int", owner=f), + "a": int, + "b": [ + support.EqualToForwardRef("builtins.undef", owner=f), + # We can't resolve this because we have to evaluate the whole annotation + support.EqualToForwardRef("builtins.int", owner=f), + ], + }, + ) + + self.assertEqual( + annotationlib.get_annotations(f, format=Format.STRING), + { + "x": "builtins.undef", + "y": "list[int]", + "z": "1 + int", + "a": "builtins.int", + "b": "[builtins.undef, builtins.int]", + }, + ) + class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): diff --git a/Misc/NEWS.d/next/Library/2025-04-22-16-35-37.gh-issue-125618.PEocn3.rst b/Misc/NEWS.d/next/Library/2025-04-22-16-35-37.gh-issue-125618.PEocn3.rst new file mode 100644 index 00000000000000..42ecf5c558fecf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-22-16-35-37.gh-issue-125618.PEocn3.rst @@ -0,0 +1,3 @@ +Add a *format* parameter to :meth:`annotationlib.ForwardRef.evaluate`. +Evaluating annotations in the ``FORWARDREF`` format now succeeds in more +cases that would previously have raised an exception. From 8b776d0025b2c839422f8338181d5e35987dd60e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Apr 2025 16:50:30 -0700 Subject: [PATCH 2/7] one line --- Doc/library/annotationlib.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 16d7e71bbcd1b4..c31a551275240f 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -172,8 +172,7 @@ Classes :class:`~ForwardRef`. The string may not be exactly equivalent to the original source. - .. method:: evaluate(*, owner=None, globals=None, locals=None, type_params=None, - format=Format.VALUE) + .. method:: evaluate(*, owner=None, globals=None, locals=None, type_params=None, format=Format.VALUE) Evaluate the forward reference, returning its value. From d095e2cf480946c7f01dc1bd1e7a848015cf3fc6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 23 Apr 2025 07:55:33 -0700 Subject: [PATCH 3/7] fix --- Lib/annotationlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 748baa495a51c1..63bba86eaa8ad5 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -676,7 +676,7 @@ def _build_closure(annotate, owner, is_class, stringifier_dict, *, owner=owner, globals=annotate.__globals__, is_class=is_class, - stringifier_dict=globals, + stringifier_dict=stringifier_dict, ) stringifier_dict.stringifiers.append(fwdref) new_cell = types.CellType(fwdref) From d27ffd8943926411b38fb9ff7504e16d4fa1b3d9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 23 Apr 2025 07:57:55 -0700 Subject: [PATCH 4/7] add test --- Lib/test/test_annotationlib.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 2c3e04df398bf9..0d00949a533cba 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1124,6 +1124,13 @@ def f( }, ) + def test_partial_evaluation_cell(self): + obj = object() + class RaisesAttributeError: + attriberr: obj.missing + anno = get_annotations(RaisesAttributeError, format=Format.FORWARDREF) + self.assertEqual(anno, None) + class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): From bf260fb7b99e817a3d4bfda31aa1f05d2928e6a4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 23 Apr 2025 08:15:53 -0700 Subject: [PATCH 5/7] fix test --- Lib/test/test_annotationlib.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 0d00949a533cba..c3218110aaf04d 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1086,7 +1086,7 @@ def test_pep_695_generics_with_future_annotations_nested_in_function(self): set(results.generic_func.__type_params__), ) - maxDiff = None + maxDiff = None def test_partial_evaluation(self): def f( @@ -1126,10 +1126,19 @@ def f( def test_partial_evaluation_cell(self): obj = object() + class RaisesAttributeError: attriberr: obj.missing + anno = get_annotations(RaisesAttributeError, format=Format.FORWARDREF) - self.assertEqual(anno, None) + self.assertEqual( + anno, + { + "attriberr": support.EqualToForwardRef( + "obj.missing", is_class=True, owner=RaisesAttributeError + ) + }, + ) class TestCallEvaluateFunction(unittest.TestCase): From 28be691c03e303ce6c1e09a0bda7f75f587c5695 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 23 Apr 2025 08:17:38 -0700 Subject: [PATCH 6/7] formatting --- Lib/annotationlib.py | 22 +++++++++++++++------- Lib/test/test_annotationlib.py | 2 -- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 63bba86eaa8ad5..471cb9e7eb4083 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -90,8 +90,15 @@ def __init__( def __init_subclass__(cls, /, *args, **kwds): raise TypeError("Cannot subclass ForwardRef") - def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None, - format=Format.VALUE): + def evaluate( + self, + *, + globals=None, + locals=None, + type_params=None, + owner=None, + format=Format.VALUE, + ): """Evaluate the forward reference and return the value. If the forward reference cannot be evaluated, raise an exception. @@ -182,8 +189,10 @@ def evaluate(self, *, globals=None, locals=None, type_params=None, owner=None, if not is_forwardref_format: raise new_locals = _StringifierDict( - {**builtins.__dict__, **locals}, globals=globals, owner=owner, - is_class=self.__forward_is_class__ + {**builtins.__dict__, **locals}, + globals=globals, + owner=owner, + is_class=self.__forward_is_class__, ) try: result = eval(code, globals=globals, locals=new_locals) @@ -650,8 +659,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): raise ValueError(f"Invalid format: {format!r}") -def _build_closure(annotate, owner, is_class, stringifier_dict, *, - allow_evaluation): +def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): if not annotate.__closure__: return None freevars = annotate.__code__.co_freevars @@ -789,7 +797,7 @@ def get_annotations( # But if we didn't get it, we use __annotations__ instead. ann = _get_dunder_annotations(obj) if ann is not None: - return annotations_to_string(ann) + return annotations_to_string(ann) case Format.VALUE_WITH_FAKE_GLOBALS: raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") case _: diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index c3218110aaf04d..3dddc308b863e5 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1086,8 +1086,6 @@ def test_pep_695_generics_with_future_annotations_nested_in_function(self): set(results.generic_func.__type_params__), ) - maxDiff = None - def test_partial_evaluation(self): def f( x: builtins.undef, From 9684e406c7483f32b49cf062b88a0211e276d910 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 4 May 2025 09:02:17 -0700 Subject: [PATCH 7/7] fix merge --- Lib/annotationlib.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 090cca745621cb..5ad0893106a72b 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -197,6 +197,7 @@ def evaluate( globals=globals, owner=owner, is_class=self.__forward_is_class__, + format=format, ) try: result = eval(code, globals=globals, locals=new_locals) @@ -703,7 +704,13 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # Try again, but do not provide any globals. This allows us to return # a value in certain cases where an exception gets raised during evaluation. - globals = _StringifierDict({}, annotate.__globals__, owner, is_class) + globals = _StringifierDict( + {}, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) closure = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False )