From 8dbbbcd356b8e9e0d83620a91f056c12cb98d9dc Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 29 Jan 2025 09:53:53 -0500 Subject: [PATCH 01/12] gh-129250: allow pickle instances of generic classes --- Include/internal/pycore_typevarobject.h | 1 + Lib/test/test_type_params.py | 29 +++ Lib/typing.py | 4 + Objects/funcobject.c | 12 +- Objects/typevarobject.c | 231 +++++++++++++++++++++++- Python/bltinmodule.c | 35 ++-- 6 files changed, 287 insertions(+), 25 deletions(-) diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index 4d7556e68cdaee..4b84f899132baf 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -17,6 +17,7 @@ extern PyObject *_Py_set_typeparam_default(PyThreadState *, PyObject *, PyObject extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); extern int _Py_typing_type_repr(PyUnicodeWriter *, PyObject *); +extern int _Py_set_type_params_owner_maybe(PyObject *, PyObject *); extern PyTypeObject _PyTypeAlias_Type; extern PyTypeObject _PyNoDefault_Type; diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 0f393def827271..963f996d9d2020 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1183,11 +1183,14 @@ def func1[X](x: X) -> X: ... def func2[X, Y](x: X | Y) -> X | Y: ... def func3[X, *Y, **Z](x: X, y: tuple[*Y], z: Z) -> X: ... def func4[X: int, Y: (bytes, str)](x: X, y: Y) -> X | Y: ... +def func3b[X, *Y, **Z]() -> X: return Class3[X, Y, Z]() +def func5[Baz](): return Class5[Baz]() class Class1[X]: ... class Class2[X, Y]: ... class Class3[X, *Y, **Z]: ... class Class4[X: int, Y: (bytes, str)]: ... +class Class5(Generic[T]): pass class TypeParamsPickleTest(unittest.TestCase): @@ -1240,6 +1243,32 @@ def test_pickling_classes(self): # but class check is good enough: self.assertIsInstance(pickle.loads(pickled), real_class) + def test_pickling_anonymous_typeparams(self): + # see gh-129250 + thing = func5() + pickled = pickle.dumps(thing) + unpickled = pickle.loads(pickled) + self.assertIs(unpickled.__orig_class__, thing.__orig_class__) + self.assertIs(unpickled.__orig_class__.__args__[0], + func5.__type_params__[0]) + + thing = func3b() + pickled = pickle.dumps(thing) + unpickled = pickle.loads(pickled) + self.assertIs(unpickled.__orig_class__, thing.__orig_class__) + self.assertIs(unpickled.__orig_class__.__args__[0], + func3b.__type_params__[0]) + self.assertIs(unpickled.__orig_class__.__args__[1], + func3b.__type_params__[1]) + self.assertIs(unpickled.__orig_class__.__args__[2], + func3b.__type_params__[2]) + + for i in range(3): + thing = Class3.__type_params__[i] + pickled = pickle.dumps(thing) + unpickled = pickle.loads(pickled) + self.assertIs(unpickled, thing) + class TypeParamsWeakRefTest(unittest.TestCase): def test_weakrefs(self): diff --git a/Lib/typing.py b/Lib/typing.py index 66570db7a5bd74..bb92b144e401b8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -502,6 +502,10 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f return t +def _restore_anonymous_typeparam(owner, index): + return owner.__type_params__[index] + + class _Final: """Mixin to prohibit subclassing.""" diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 7b17a9ba31fac4..32f67001e5f5cf 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -2,11 +2,12 @@ /* Function object implementation */ #include "Python.h" -#include "pycore_dict.h" // _Py_INCREF_DICT() -#include "pycore_long.h" // _PyLong_GetOne() -#include "pycore_modsupport.h" // _PyArg_NoKeywords() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_Occurred() +#include "pycore_dict.h" // _Py_INCREF_DICT() +#include "pycore_long.h" // _PyLong_GetOne() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_Occurred() +#include "pycore_typevarobject.h" // _Py_set_type_params_owner_maybe() static const char * @@ -926,6 +927,7 @@ _Py_set_function_type_params(PyThreadState *Py_UNUSED(ignored), PyObject *func, assert(PyFunction_Check(func)); assert(PyTuple_Check(type_params)); PyFunctionObject *f = (PyFunctionObject *)func; + _Py_set_type_params_owner_maybe(type_params, func); Py_XSETREF(f->func_typeparams, Py_NewRef(type_params)); return Py_NewRef(func); } diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 4ed40aa71a595e..9b716702943ca1 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -25,6 +25,7 @@ typedef struct { PyObject *evaluate_constraints; PyObject *default_value; PyObject *evaluate_default; + PyObject *owner; bool covariant; bool contravariant; bool infer_variance; @@ -35,6 +36,7 @@ typedef struct { PyObject *name; PyObject *default_value; PyObject *evaluate_default; + PyObject *owner; } typevartupleobject; typedef struct { @@ -43,6 +45,7 @@ typedef struct { PyObject *bound; PyObject *default_value; PyObject *evaluate_default; + PyObject *owner; bool covariant; bool contravariant; bool infer_variance; @@ -449,6 +452,84 @@ unpack_typevartuples(PyObject *params) } } +static PyObject * +typeparam_reduce_anonymous(PyObject *self, PyObject *wr_owner) +{ + PyObject *ret = NULL; + PyObject *args = NULL; + PyObject *func = NULL; + PyObject *typing = NULL; + PyObject *index = NULL; + PyObject *type_params = NULL; + PyObject *owner = NULL; + + if (PyWeakref_GetRef(wr_owner, &owner) == -1) { + goto done; + } + if (owner == NULL) { + PyErr_SetString(PyExc_RuntimeError, "owner of this type param is dead"); + goto done; + } + type_params = PyObject_GetAttr(owner, &_Py_ID(__type_params__)); + if (type_params == NULL) { + goto done; + } + if (!PyTuple_Check(type_params)) { + PyErr_SetString(PyExc_ValueError, "expecting __type_params__ to be tuple"); + goto done; + } + Py_ssize_t size = PyTuple_GET_SIZE(type_params); + Py_ssize_t i = 0; + for (; i < size; i++) { + if (PyTuple_GET_ITEM(type_params, i) == self) { + break; + } + } + if (i == size) { + PyErr_SetString(PyExc_ValueError, "type param not found in owner __type_params__"); + goto done; + } + index = PyLong_FromLong(i); + if (index == NULL) { + goto done; + } + typing = PyImport_ImportModule("typing"); + if (typing == NULL) { + goto done; + } + func = PyObject_GetAttrString(typing, "_restore_anonymous_typeparam"); + if (func == NULL) { + goto done; + } + + args = PyTuple_New(2); + if (args == NULL) { + goto done; + } + ret = PyTuple_New(2); + if (ret == NULL) { + goto done; + } + PyTuple_SET_ITEM(args, 0, owner); + PyTuple_SET_ITEM(args, 1, index); + PyTuple_SET_ITEM(ret, 0, func); + PyTuple_SET_ITEM(ret, 1, args); + args = NULL; + func = NULL; + index = NULL; + owner = NULL; + +done: + Py_XDECREF(args); + Py_XDECREF(func); + Py_XDECREF(typing); + Py_XDECREF(index); + Py_XDECREF(type_params); + Py_XDECREF(owner); + + return ret; +} + static void typevar_dealloc(PyObject *self) { @@ -464,6 +545,7 @@ typevar_dealloc(PyObject *self) Py_XDECREF(tv->evaluate_constraints); Py_XDECREF(tv->default_value); Py_XDECREF(tv->evaluate_default); + Py_XDECREF(tv->owner); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -482,6 +564,7 @@ typevar_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(tv->evaluate_constraints); Py_VISIT(tv->default_value); Py_VISIT(tv->evaluate_default); + Py_VISIT(tv->owner); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -495,6 +578,7 @@ typevar_clear(typevarobject *self) Py_CLEAR(self->evaluate_constraints); Py_CLEAR(self->default_value); Py_CLEAR(self->evaluate_default); + Py_CLEAR(self->owner); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -630,6 +714,7 @@ typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, tv->evaluate_constraints = Py_XNewRef(evaluate_constraints); tv->default_value = Py_XNewRef(default_value); tv->evaluate_default = NULL; + tv->owner = NULL; tv->covariant = covariant; tv->contravariant = contravariant; @@ -806,6 +891,10 @@ static PyObject * typevar_reduce_impl(typevarobject *self) /*[clinic end generated code: output=02e5c55d7cf8a08f input=de76bc95f04fb9ff]*/ { + PyObject *wr_owner = (PyObject *)FT_ATOMIC_LOAD_PTR(self->owner); + if (wr_owner != NULL) { + return typeparam_reduce_anonymous((PyObject *)self, wr_owner); + } return Py_NewRef(self->name); } @@ -1153,6 +1242,7 @@ paramspec_dealloc(PyObject *self) Py_XDECREF(ps->bound); Py_XDECREF(ps->default_value); Py_XDECREF(ps->evaluate_default); + Py_XDECREF(ps->owner); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -1168,6 +1258,7 @@ paramspec_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(ps->bound); Py_VISIT(ps->default_value); Py_VISIT(ps->evaluate_default); + Py_VISIT(ps->owner); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -1178,6 +1269,7 @@ paramspec_clear(paramspecobject *self) Py_CLEAR(self->bound); Py_CLEAR(self->default_value); Py_CLEAR(self->evaluate_default); + Py_CLEAR(self->owner); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -1268,6 +1360,7 @@ paramspec_alloc(PyObject *name, PyObject *bound, PyObject *default_value, bool c ps->infer_variance = infer_variance; ps->default_value = Py_XNewRef(default_value); ps->evaluate_default = NULL; + ps->owner = NULL; _PyObject_GC_TRACK(ps); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)ps, "__module__", module) < 0) { @@ -1372,6 +1465,10 @@ static PyObject * paramspec_reduce_impl(paramspecobject *self) /*[clinic end generated code: output=b83398674416db27 input=5bf349f0d5dd426c]*/ { + PyObject *wr_owner = (PyObject *)FT_ATOMIC_LOAD_PTR(self->owner); + if (wr_owner != NULL) { + return typeparam_reduce_anonymous((PyObject *)self, wr_owner); + } return Py_NewRef(self->name); } @@ -1456,9 +1553,6 @@ Parameter specification variables can be introspected. e.g.::\n\ >>> P = ParamSpec(\"P\")\n\ >>> P.__name__\n\ 'P'\n\ -\n\ -Note that only parameter specification variables defined in the global\n\ -scope can be pickled.\n\ "); static PyType_Slot paramspec_slots[] = { @@ -1497,6 +1591,7 @@ typevartuple_dealloc(PyObject *self) Py_DECREF(tvt->name); Py_XDECREF(tvt->default_value); Py_XDECREF(tvt->evaluate_default); + Py_XDECREF(tvt->owner); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -1546,6 +1641,7 @@ typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) tvt->name = Py_NewRef(name); tvt->default_value = Py_XNewRef(default_value); tvt->evaluate_default = NULL; + tvt->owner = NULL; _PyObject_GC_TRACK(tvt); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)tvt, "__module__", module) < 0) { @@ -1626,6 +1722,10 @@ static PyObject * typevartuple_reduce_impl(typevartupleobject *self) /*[clinic end generated code: output=3215bc0477913d20 input=3018a4d66147e807]*/ { + PyObject *wr_owner = (PyObject *)FT_ATOMIC_LOAD_PTR(self->owner); + if (wr_owner != NULL) { + return typeparam_reduce_anonymous((PyObject *)self, wr_owner); + } return Py_NewRef(self->name); } @@ -1660,6 +1760,7 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); Py_VISIT(((typevartupleobject *)self)->default_value); Py_VISIT(((typevartupleobject *)self)->evaluate_default); + Py_VISIT(((typevartupleobject *)self)->owner); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -1669,6 +1770,7 @@ typevartuple_clear(PyObject *self) { Py_CLEAR(((typevartupleobject *)self)->default_value); Py_CLEAR(((typevartupleobject *)self)->evaluate_default); + Py_CLEAR(((typevartupleobject *)self)->owner); PyObject_ClearManagedDict(self); return 0; } @@ -1749,9 +1851,6 @@ arguments::\n\ C[()] # Even this is fine\n\ \n\ For more details, see PEP 646.\n\ -\n\ -Note that only TypeVarTuples defined in the global scope can be\n\ -pickled.\n\ "); PyType_Slot typevartuple_slots[] = { @@ -2347,3 +2446,123 @@ _Py_set_typeparam_default(PyThreadState *ts, PyObject *typeparam, PyObject *eval return NULL; } } + +static int +_set_typeparam_owner_maybe(PyThreadState *ts, PyObject *typeparam, + PyObject *owner, PyObject *name, PyObject *curowner, + PyObject **wr_owner) +{ + *wr_owner = NULL; + if (curowner != NULL) { + // owner can only be assigned once + return 0; + } + + PyObject *module; + if (!PyObject_GetOptionalAttr((PyObject *)typeparam, &_Py_ID(__module__), + &module)) { + return -1; + } + int res = _PyUnicode_EqualToASCIIString(module, "typing"); + Py_DECREF(module); + if (!res) { + return 0; + } + + PyObject *typing = PyImport_ImportModule("typing"); + if (typing == NULL) { + PyErr_Clear(); + return -1; + } + PyObject *attr; + res = PyObject_GetOptionalAttr(typing, name, &attr); + Py_DECREF(typing); + if (res == -1) { + PyErr_Clear(); + return -1; + } + Py_XDECREF(attr); + if (attr == typeparam) { + // name exists in typing module and is this typeparam + return 0; + } + + *wr_owner = PyWeakref_NewRef(owner, NULL); + if (*wr_owner == NULL) { + PyErr_Clear(); + return -1; + } + return 1; +} + +static int +set_typeparam_owner_maybe(PyThreadState *ts, PyObject *typeparam, + PyObject *owner) +{ + int ret; + PyObject *wr_owner; + + if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { + ret = _set_typeparam_owner_maybe(ts, typeparam, owner, + ((typevarobject *)typeparam)->name, + FT_ATOMIC_LOAD_PTR(((typevarobject *)typeparam)->owner), + &wr_owner); + if (ret == 1) { +#ifdef Py_GIL_DISABLED + wr_owner = _Py_atomic_exchange_ptr(&((typevarobject *)typeparam)->owner, + wr_owner); + Py_XDECREF(wr_owner); +#else + ((typevarobject *)typeparam)->owner = wr_owner; +#endif + } + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { + ret = _set_typeparam_owner_maybe(ts, typeparam, owner, + ((paramspecobject *)typeparam)->name, + FT_ATOMIC_LOAD_PTR(((paramspecobject *)typeparam)->owner), + &wr_owner); + if (ret == 1) { +#ifdef Py_GIL_DISABLED + wr_owner = _Py_atomic_exchange_ptr(&((paramspecobject *)typeparam)->owner, + wr_owner); + Py_XDECREF(wr_owner); +#else + ((paramspecobject *)typeparam)->owner = wr_owner; +#endif + } + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { + ret = _set_typeparam_owner_maybe(ts, typeparam, owner, + ((typevartupleobject *)typeparam)->name, + FT_ATOMIC_LOAD_PTR(((typevartupleobject *)typeparam)->owner), + &wr_owner); + if (ret == 1) { +#ifdef Py_GIL_DISABLED + wr_owner = _Py_atomic_exchange_ptr(&((typevartupleobject *)typeparam)->owner, + wr_owner); + Py_XDECREF(wr_owner); +#else + ((typevartupleobject *)typeparam)->owner = wr_owner; +#endif + } + } + else { + return -1; + } + return ret; +} + +int +_Py_set_type_params_owner_maybe(PyObject *type_params, PyObject *owner) +{ + assert(PyTuple_Check(type_params)); + PyThreadState *ts = _PyThreadState_GET(); + assert(ts != NULL); + int ret = 0; + for (Py_ssize_t i = 0; i < Py_SIZE(type_params); i++) { + ret |= set_typeparam_owner_maybe(ts, PyTuple_GET_ITEM(type_params, i), + owner); + } + return ret; +} diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 46a6fd9a8ef017..35a15e22210bc1 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1,24 +1,25 @@ /* Built-in functions */ #include "Python.h" -#include "pycore_ast.h" // _PyAST_Validate() -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_Vector() -#include "pycore_compile.h" // _PyAST_Compile() -#include "pycore_long.h" // _PyLong_CompactValue -#include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _Py_AddToAllObjects() -#include "pycore_pyerrors.h" // _PyErr_NoMemory() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_pythonrun.h" // _Py_SourceAsString() -#include "pycore_sysmodule.h" // _PySys_GetAttr() -#include "pycore_tuple.h" // _PyTuple_FromArray() -#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_ast.h" // _PyAST_Validate() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_Vector() +#include "pycore_compile.h" // _PyAST_Compile() +#include "pycore_long.h" // _PyLong_CompactValue +#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_object.h" // _Py_AddToAllObjects() +#include "pycore_pyerrors.h" // _PyErr_NoMemory() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_pythonrun.h" // _Py_SourceAsString() +#include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_typevarobject.h" // _Py_set_type_params_owner_maybe() +#include "pycore_cell.h" // PyCell_GetRef() #include "clinic/bltinmodule.c.h" #ifdef HAVE_UNISTD_H -# include // isatty() +# include // isatty() #endif @@ -230,6 +231,12 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py_DECREF(cell_cls); } } + if (cls != NULL) { + PyObject *type_params = PyDict_GetItem(ns, &_Py_ID(__type_params__)); + if (type_params != NULL) { + _Py_set_type_params_owner_maybe(type_params, cls); + } + } } error: Py_XDECREF(cell); From 4173ce0f0d05506aba300f8e4321621ccf210f81 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 29 Jan 2025 15:32:04 +0000 Subject: [PATCH 02/12] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst new file mode 100644 index 00000000000000..490fe1c35c150c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst @@ -0,0 +1 @@ +Allow pickle of instances of generic classes. From dad90ca54b06f95405270b5910cca28842192e96 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 29 Jan 2025 13:54:42 -0500 Subject: [PATCH 03/12] PyLong_FromLong -> PyLong_FromSsize_t --- Objects/typevarobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 9b716702943ca1..04a4a2dcc14252 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -489,7 +489,7 @@ typeparam_reduce_anonymous(PyObject *self, PyObject *wr_owner) PyErr_SetString(PyExc_ValueError, "type param not found in owner __type_params__"); goto done; } - index = PyLong_FromLong(i); + index = PyLong_FromSsize_t(i); if (index == NULL) { goto done; } From f7cc7c63dea564804a2818cdacbab64d7b3df73e Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 30 Jan 2025 13:16:47 -0500 Subject: [PATCH 04/12] remove weakref, owner stored as (mod, name, idx) --- Doc/library/dis.rst | 83 +++---- Include/internal/pycore_function.h | 2 +- Include/internal/pycore_intrinsics.h | 3 +- Include/internal/pycore_typevarobject.h | 2 +- Objects/funcobject.c | 8 +- Objects/typevarobject.c | 275 +++++++++++------------- Python/bltinmodule.c | 35 ++- Python/codegen.c | 4 + Python/intrinsics.c | 1 + 9 files changed, 200 insertions(+), 213 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index f8f4188d27b472..5eafda817a5229 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1778,45 +1778,50 @@ iterations of the loop. The operand determines which intrinsic function is called: - +-----------------------------------+-----------------------------------+ - | Operand | Description | - +===================================+===================================+ - | ``INTRINSIC_1_INVALID`` | Not valid | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_PRINT`` | Prints the argument to standard | - | | out. Used in the REPL. | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_IMPORT_STAR`` | Performs ``import *`` for the | - | | named module. | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_STOPITERATION_ERROR`` | Extracts the return value from a | - | | ``StopIteration`` exception. | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an async generator value | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_UNARY_POSITIVE`` | Performs the unary ``+`` | - | | operation | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_LIST_TO_TUPLE`` | Converts a list to a tuple | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_TYPEVAR`` | Creates a :class:`typing.TypeVar` | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_PARAMSPEC`` | Creates a | - | | :class:`typing.ParamSpec` | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_TYPEVARTUPLE`` | Creates a | - | | :class:`typing.TypeVarTuple` | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_SUBSCRIPT_GENERIC`` | Returns :class:`typing.Generic` | - | | subscripted with the argument | - +-----------------------------------+-----------------------------------+ - | ``INTRINSIC_TYPEALIAS`` | Creates a | - | | :class:`typing.TypeAliasType`; | - | | used in the :keyword:`type` | - | | statement. The argument is a tuple| - | | of the type alias's name, | - | | type parameters, and value. | - +-----------------------------------+-----------------------------------+ + +-------------------------------------+-----------------------------------+ + | Operand | Description | + +=====================================+===================================+ + | ``INTRINSIC_1_INVALID`` | Not valid | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_PRINT`` | Prints the argument to standard | + | | out. Used in the REPL. | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_IMPORT_STAR`` | Performs ``import *`` for the | + | | named module. | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_STOPITERATION_ERROR`` | Extracts the return value from a | + | | ``StopIteration`` exception. | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an async generator value | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_UNARY_POSITIVE`` | Performs the unary ``+`` | + | | operation | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_LIST_TO_TUPLE`` | Converts a list to a tuple | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVAR`` | Creates a :class:`typing.TypeVar` | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_PARAMSPEC`` | Creates a | + | | :class:`typing.ParamSpec` | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVARTUPLE`` | Creates a | + | | :class:`typing.TypeVarTuple` | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_SUBSCRIPT_GENERIC`` | Returns :class:`typing.Generic` | + | | subscripted with the argument | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEALIAS`` | Creates a | + | | :class:`typing.TypeAliasType`; | + | | used in the :keyword:`type` | + | | statement. The argument is a tuple| + | | of the type alias's name, | + | | type parameters, and value. | + +-------------------------------------+-----------------------------------+ + | ``INTRINSIC_SET_TYPE_PARAMS_OWNER`` | Sets owner for any generics | + | | stored in an object's | + | | ``__type_params__`` to be that | + | | object. | + +-------------------------------------+-----------------------------------+ .. versionadded:: 3.12 diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index c45d281125febb..26706ee6ddf590 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -57,7 +57,7 @@ void _PyFunction_ClearCodeByVersion(uint32_t version); PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code); extern PyObject *_Py_set_function_type_params( - PyThreadState* unused, PyObject *func, PyObject *type_params); + PyThreadState *ts, PyObject *func, PyObject *type_params); #ifdef __cplusplus } diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 39c2a30f6e979d..983c403d2c51f6 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -18,8 +18,9 @@ #define INTRINSIC_TYPEVARTUPLE 9 #define INTRINSIC_SUBSCRIPT_GENERIC 10 #define INTRINSIC_TYPEALIAS 11 +#define INTRINSIC_SET_TYPE_PARAMS_OWNER 12 -#define MAX_INTRINSIC_1 11 +#define MAX_INTRINSIC_1 12 /* Binary Functions: */ diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index 4b84f899132baf..dbf770f8bb29bc 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -14,10 +14,10 @@ extern PyObject *_Py_make_typevartuple(PyThreadState *, PyObject *); extern PyObject *_Py_make_typealias(PyThreadState *, PyObject *); extern PyObject *_Py_subscript_generic(PyThreadState *, PyObject *); extern PyObject *_Py_set_typeparam_default(PyThreadState *, PyObject *, PyObject *); +extern PyObject *_Py_set_type_params_owner(PyThreadState *, PyObject *); extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); extern int _Py_typing_type_repr(PyUnicodeWriter *, PyObject *); -extern int _Py_set_type_params_owner_maybe(PyObject *, PyObject *); extern PyTypeObject _PyTypeAlias_Type; extern PyTypeObject _PyNoDefault_Type; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 32f67001e5f5cf..7fb65da3a724d7 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -7,7 +7,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() -#include "pycore_typevarobject.h" // _Py_set_type_params_owner_maybe() +#include "pycore_typevarobject.h" // _Py_set_type_params_owner() static const char * @@ -921,14 +921,16 @@ func_set_type_params(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) } PyObject * -_Py_set_function_type_params(PyThreadState *Py_UNUSED(ignored), PyObject *func, +_Py_set_function_type_params(PyThreadState *ts, PyObject *func, PyObject *type_params) { assert(PyFunction_Check(func)); assert(PyTuple_Check(type_params)); PyFunctionObject *f = (PyFunctionObject *)func; - _Py_set_type_params_owner_maybe(type_params, func); Py_XSETREF(f->func_typeparams, Py_NewRef(type_params)); + if (_Py_set_type_params_owner(ts, func) == NULL) { + return NULL; + } return Py_NewRef(func); } diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 04a4a2dcc14252..4db48255199ead 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -453,80 +453,76 @@ unpack_typevartuples(PyObject *params) } static PyObject * -typeparam_reduce_anonymous(PyObject *self, PyObject *wr_owner) -{ +typeparam_reduce_anonymous(PyObject *self, PyObject *owner) +{ + assert(PyTuple_Check(owner) && PyTuple_GET_SIZE(owner) == 3); + PyObject *module_name = PyTuple_GET_ITEM(owner, 0); + PyObject *qualname = PyTuple_GET_ITEM(owner, 1); + PyObject *index = PyTuple_GET_ITEM(owner, 2); + assert(PyUnicode_Check(module_name)); + assert(PyUnicode_Check(qualname)); + assert(PyLong_Check(index)); PyObject *ret = NULL; - PyObject *args = NULL; - PyObject *func = NULL; PyObject *typing = NULL; - PyObject *index = NULL; - PyObject *type_params = NULL; - PyObject *owner = NULL; + PyObject *restore_func = NULL; + PyObject *module = NULL; + PyObject *path = NULL; + PyObject *obj = NULL; + PyObject *args = NULL; - if (PyWeakref_GetRef(wr_owner, &owner) == -1) { + typing = PyImport_ImportModule("typing"); + if (typing == NULL) { goto done; } - if (owner == NULL) { - PyErr_SetString(PyExc_RuntimeError, "owner of this type param is dead"); + restore_func = PyObject_GetAttrString(typing, "_restore_anonymous_typeparam"); + if (restore_func == NULL) { goto done; } - type_params = PyObject_GetAttr(owner, &_Py_ID(__type_params__)); - if (type_params == NULL) { + + module = PyImport_Import(module_name); + if (module == NULL) { goto done; } - if (!PyTuple_Check(type_params)) { - PyErr_SetString(PyExc_ValueError, "expecting __type_params__ to be tuple"); + PyObject *attrsep = PyUnicode_FromString("."); + if (attrsep == NULL) { goto done; } - Py_ssize_t size = PyTuple_GET_SIZE(type_params); - Py_ssize_t i = 0; - for (; i < size; i++) { - if (PyTuple_GET_ITEM(type_params, i) == self) { - break; - } - } - if (i == size) { - PyErr_SetString(PyExc_ValueError, "type param not found in owner __type_params__"); + path = PyUnicode_Split(qualname, attrsep, -1); + Py_DECREF(attrsep); + if (path == NULL) { goto done; } - index = PyLong_FromSsize_t(i); - if (index == NULL) { + if (PyList_GET_SIZE(path) == 0) { + PyErr_SetString(PyExc_ValueError, "invalid typeparam owner qualname"); goto done; } - typing = PyImport_ImportModule("typing"); - if (typing == NULL) { - goto done; - } - func = PyObject_GetAttrString(typing, "_restore_anonymous_typeparam"); - if (func == NULL) { - goto done; + obj = module; + Py_INCREF(obj); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(path); i++) { + PyObject *parent = obj; + obj = PyObject_GetAttr(parent, PyList_GET_ITEM(path, i)); + Py_DECREF(parent); + if (obj == NULL) { + goto done; + } } - args = PyTuple_New(2); + args = PyTuple_Pack(2, obj, index); if (args == NULL) { goto done; } - ret = PyTuple_New(2); + ret = PyTuple_Pack(2, restore_func, args); if (ret == NULL) { goto done; } - PyTuple_SET_ITEM(args, 0, owner); - PyTuple_SET_ITEM(args, 1, index); - PyTuple_SET_ITEM(ret, 0, func); - PyTuple_SET_ITEM(ret, 1, args); - args = NULL; - func = NULL; - index = NULL; - owner = NULL; done: Py_XDECREF(args); - Py_XDECREF(func); + Py_XDECREF(obj); + Py_XDECREF(path); + Py_XDECREF(module); + Py_XDECREF(restore_func); Py_XDECREF(typing); - Py_XDECREF(index); - Py_XDECREF(type_params); - Py_XDECREF(owner); - return ret; } @@ -891,9 +887,9 @@ static PyObject * typevar_reduce_impl(typevarobject *self) /*[clinic end generated code: output=02e5c55d7cf8a08f input=de76bc95f04fb9ff]*/ { - PyObject *wr_owner = (PyObject *)FT_ATOMIC_LOAD_PTR(self->owner); - if (wr_owner != NULL) { - return typeparam_reduce_anonymous((PyObject *)self, wr_owner); + PyObject *owner = (PyObject *)FT_ATOMIC_LOAD_PTR(self->owner); + if (owner != NULL) { + return typeparam_reduce_anonymous((PyObject *)self, owner); } return Py_NewRef(self->name); } @@ -976,6 +972,8 @@ be explicitly marked covariant or contravariant by passing\n\ ``covariant=True`` or ``contravariant=True``. By default, manually\n\ created type variables are invariant. See PEP 484 and PEP 695 for more\n\ details.\n\ +\n\ +Note that only TypeVars reachable from the global scope can be pickled.\n\ "); static PyType_Slot typevar_slots[] = { @@ -1465,9 +1463,9 @@ static PyObject * paramspec_reduce_impl(paramspecobject *self) /*[clinic end generated code: output=b83398674416db27 input=5bf349f0d5dd426c]*/ { - PyObject *wr_owner = (PyObject *)FT_ATOMIC_LOAD_PTR(self->owner); - if (wr_owner != NULL) { - return typeparam_reduce_anonymous((PyObject *)self, wr_owner); + PyObject *owner = (PyObject *)FT_ATOMIC_LOAD_PTR(self->owner); + if (owner != NULL) { + return typeparam_reduce_anonymous((PyObject *)self, owner); } return Py_NewRef(self->name); } @@ -1553,6 +1551,9 @@ Parameter specification variables can be introspected. e.g.::\n\ >>> P = ParamSpec(\"P\")\n\ >>> P.__name__\n\ 'P'\n\ +\n\ +Note that only parameter specification variables reachable from the\n\ +scope can be pickled.\n\ "); static PyType_Slot paramspec_slots[] = { @@ -1722,9 +1723,9 @@ static PyObject * typevartuple_reduce_impl(typevartupleobject *self) /*[clinic end generated code: output=3215bc0477913d20 input=3018a4d66147e807]*/ { - PyObject *wr_owner = (PyObject *)FT_ATOMIC_LOAD_PTR(self->owner); - if (wr_owner != NULL) { - return typeparam_reduce_anonymous((PyObject *)self, wr_owner); + PyObject *owner = (PyObject *)FT_ATOMIC_LOAD_PTR(self->owner); + if (owner != NULL) { + return typeparam_reduce_anonymous((PyObject *)self, owner); } return Py_NewRef(self->name); } @@ -1851,6 +1852,9 @@ arguments::\n\ C[()] # Even this is fine\n\ \n\ For more details, see PEP 646.\n\ +\n\ +Note that only TypeVarTuples reachable from the global scope can be\n\ +pickled.\n\ "); PyType_Slot typevartuple_slots[] = { @@ -2448,121 +2452,98 @@ _Py_set_typeparam_default(PyThreadState *ts, PyObject *typeparam, PyObject *eval } static int -_set_typeparam_owner_maybe(PyThreadState *ts, PyObject *typeparam, - PyObject *owner, PyObject *name, PyObject *curowner, - PyObject **wr_owner) +set_typeparam_owner(PyThreadState *ts, PyObject *typeparam, PyObject *owner) { - *wr_owner = NULL; - if (curowner != NULL) { - // owner can only be assigned once - return 0; - } - - PyObject *module; - if (!PyObject_GetOptionalAttr((PyObject *)typeparam, &_Py_ID(__module__), - &module)) { - return -1; - } - int res = _PyUnicode_EqualToASCIIString(module, "typing"); - Py_DECREF(module); - if (!res) { - return 0; - } - - PyObject *typing = PyImport_ImportModule("typing"); - if (typing == NULL) { - PyErr_Clear(); - return -1; - } - PyObject *attr; - res = PyObject_GetOptionalAttr(typing, name, &attr); - Py_DECREF(typing); - if (res == -1) { - PyErr_Clear(); - return -1; - } - Py_XDECREF(attr); - if (attr == typeparam) { - // name exists in typing module and is this typeparam - return 0; - } - - *wr_owner = PyWeakref_NewRef(owner, NULL); - if (*wr_owner == NULL) { - PyErr_Clear(); - return -1; - } - return 1; -} - -static int -set_typeparam_owner_maybe(PyThreadState *ts, PyObject *typeparam, - PyObject *owner) -{ - int ret; - PyObject *wr_owner; - if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { - ret = _set_typeparam_owner_maybe(ts, typeparam, owner, - ((typevarobject *)typeparam)->name, - FT_ATOMIC_LOAD_PTR(((typevarobject *)typeparam)->owner), - &wr_owner); - if (ret == 1) { #ifdef Py_GIL_DISABLED - wr_owner = _Py_atomic_exchange_ptr(&((typevarobject *)typeparam)->owner, - wr_owner); - Py_XDECREF(wr_owner); + owner = _Py_atomic_exchange_ptr(&((typevarobject *)typeparam)->owner, owner); + assert(owner == NULL); #else - ((typevarobject *)typeparam)->owner = wr_owner; + assert(((typevarobject *)typeparam)->owner == NULL); + ((typevarobject *)typeparam)->owner = owner; #endif - } } else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { - ret = _set_typeparam_owner_maybe(ts, typeparam, owner, - ((paramspecobject *)typeparam)->name, - FT_ATOMIC_LOAD_PTR(((paramspecobject *)typeparam)->owner), - &wr_owner); - if (ret == 1) { #ifdef Py_GIL_DISABLED - wr_owner = _Py_atomic_exchange_ptr(&((paramspecobject *)typeparam)->owner, - wr_owner); - Py_XDECREF(wr_owner); + owner = _Py_atomic_exchange_ptr(&((paramspecobject *)typeparam)->owner, owner); + assert(owner == NULL); #else - ((paramspecobject *)typeparam)->owner = wr_owner; + assert(((paramspecobject *)typeparam)->owner == NULL); + ((paramspecobject *)typeparam)->owner = owner; #endif - } } else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { - ret = _set_typeparam_owner_maybe(ts, typeparam, owner, - ((typevartupleobject *)typeparam)->name, - FT_ATOMIC_LOAD_PTR(((typevartupleobject *)typeparam)->owner), - &wr_owner); - if (ret == 1) { #ifdef Py_GIL_DISABLED - wr_owner = _Py_atomic_exchange_ptr(&((typevartupleobject *)typeparam)->owner, - wr_owner); - Py_XDECREF(wr_owner); + owner = _Py_atomic_exchange_ptr(&((typevartupleobject *)typeparam)->owner, owner); + assert(owner == NULL); #else - ((typevartupleobject *)typeparam)->owner = wr_owner; + assert(((typevartupleobject *)typeparam)->owner == NULL); + ((typevartupleobject *)typeparam)->owner = owner; #endif - } } else { return -1; } - return ret; + return 0; } -int -_Py_set_type_params_owner_maybe(PyObject *type_params, PyObject *owner) +static int +set_type_params_owner(PyThreadState *ts, PyObject *type_params, PyObject *owner) { assert(PyTuple_Check(type_params)); - PyThreadState *ts = _PyThreadState_GET(); - assert(ts != NULL); - int ret = 0; + int ret = -1; + PyObject *module = NULL; + PyObject *qualname = NULL; + + int res = PyObject_GetOptionalAttr(owner, &_Py_ID(__module__), &module); + if (res <= 0) { + goto done; + } + res = PyObject_GetOptionalAttr(owner, &_Py_ID(__qualname__), &qualname); + if (res <= 0) { + goto done; + } + for (Py_ssize_t i = 0; i < Py_SIZE(type_params); i++) { - ret |= set_typeparam_owner_maybe(ts, PyTuple_GET_ITEM(type_params, i), - owner); + PyObject *index = PyLong_FromSsize_t(i); + if (index == NULL) { + goto done; + } + PyObject *owner_tuple = PyTuple_Pack(3, module, qualname, index); + Py_DECREF(index); + if (owner_tuple == NULL) { + goto done; + } + if (set_typeparam_owner(ts, PyTuple_GET_ITEM(type_params, i), owner_tuple) < 0) { + Py_DECREF(owner_tuple); + PyErr_SetString(PyExc_RuntimeError, "failed to set typeparam owner"); + goto done; + } + } + ret = 0; + +done: + Py_XDECREF(qualname); + Py_XDECREF(module); + return ret; +} + +PyObject * +_Py_set_type_params_owner(PyThreadState *ts, PyObject *owner) +{ + PyObject *ret = NULL; + PyObject *type_params = PyObject_GetAttr(owner, &_Py_ID(__type_params__)); + if (type_params == NULL) { + goto done; } + if (!PyTuple_Check(type_params)) { + PyErr_SetString(PyExc_ValueError, "expecting __type_params__ to be a tuple"); + goto done; + } + if (set_type_params_owner(ts, type_params, owner) == 0) { + ret = Py_None; + } +done: + Py_XDECREF(type_params); return ret; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 35a15e22210bc1..46a6fd9a8ef017 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1,25 +1,24 @@ /* Built-in functions */ #include "Python.h" -#include "pycore_ast.h" // _PyAST_Validate() -#include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_ceval.h" // _PyEval_Vector() -#include "pycore_compile.h" // _PyAST_Compile() -#include "pycore_long.h" // _PyLong_CompactValue -#include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _Py_AddToAllObjects() -#include "pycore_pyerrors.h" // _PyErr_NoMemory() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_pythonrun.h" // _Py_SourceAsString() -#include "pycore_sysmodule.h" // _PySys_GetAttr() -#include "pycore_tuple.h" // _PyTuple_FromArray() -#include "pycore_typevarobject.h" // _Py_set_type_params_owner_maybe() -#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_ast.h" // _PyAST_Validate() +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_ceval.h" // _PyEval_Vector() +#include "pycore_compile.h" // _PyAST_Compile() +#include "pycore_long.h" // _PyLong_CompactValue +#include "pycore_modsupport.h" // _PyArg_NoKwnames() +#include "pycore_object.h" // _Py_AddToAllObjects() +#include "pycore_pyerrors.h" // _PyErr_NoMemory() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_pythonrun.h" // _Py_SourceAsString() +#include "pycore_sysmodule.h" // _PySys_GetAttr() +#include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_cell.h" // PyCell_GetRef() #include "clinic/bltinmodule.c.h" #ifdef HAVE_UNISTD_H -# include // isatty() +# include // isatty() #endif @@ -231,12 +230,6 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py_DECREF(cell_cls); } } - if (cls != NULL) { - PyObject *type_params = PyDict_GetItem(ns, &_Py_ID(__type_params__)); - if (type_params != NULL) { - _Py_set_type_params_owner_maybe(type_params, cls); - } - } } error: Py_XDECREF(cell); diff --git a/Python/codegen.c b/Python/codegen.c index df3b5aaac1d0d9..fd3235e4e2dce4 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1595,6 +1595,10 @@ codegen_class(compiler *c, stmt_ty s) &_Py_STR(generic_base), s->v.ClassDef.keywords)); + ADDOP_I(c, loc, COPY, 1); + ADDOP_I_IN_SCOPE(c, loc, CALL_INTRINSIC_1, INTRINSIC_SET_TYPE_PARAMS_OWNER); + ADDOP(c, loc, POP_TOP); + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); _PyCompile_ExitScope(c); diff --git a/Python/intrinsics.c b/Python/intrinsics.c index a6b2c108b67175..a98a342c7742d2 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -220,6 +220,7 @@ _PyIntrinsics_UnaryFunctions[] = { INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVARTUPLE, _Py_make_typevartuple) INTRINSIC_FUNC_ENTRY(INTRINSIC_SUBSCRIPT_GENERIC, _Py_subscript_generic) INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEALIAS, _Py_make_typealias) + INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_TYPE_PARAMS_OWNER, _Py_set_type_params_owner) }; From e794467bec5133e538ef7e3b6fbd3b269a672d15 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 30 Jan 2025 17:23:23 -0500 Subject: [PATCH 05/12] misc tweaks --- Objects/funcobject.c | 4 +++- Objects/typevarobject.c | 10 +++------- Python/codegen.c | 3 --- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 7fb65da3a724d7..668cf82b2f79de 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -928,9 +928,11 @@ _Py_set_function_type_params(PyThreadState *ts, PyObject *func, assert(PyTuple_Check(type_params)); PyFunctionObject *f = (PyFunctionObject *)func; Py_XSETREF(f->func_typeparams, Py_NewRef(type_params)); - if (_Py_set_type_params_owner(ts, func) == NULL) { + PyObject *res = _Py_set_type_params_owner(ts, func); + if (res == NULL) { return NULL; } + Py_DECREF(res); return Py_NewRef(func); } diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 4db48255199ead..5446eba5011d40 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -483,12 +483,7 @@ typeparam_reduce_anonymous(PyObject *self, PyObject *owner) if (module == NULL) { goto done; } - PyObject *attrsep = PyUnicode_FromString("."); - if (attrsep == NULL) { - goto done; - } - path = PyUnicode_Split(qualname, attrsep, -1); - Py_DECREF(attrsep); + path = PyUnicode_Split(qualname, _Py_LATIN1_CHR('.'), -1); if (path == NULL) { goto done; } @@ -2541,7 +2536,8 @@ _Py_set_type_params_owner(PyThreadState *ts, PyObject *owner) goto done; } if (set_type_params_owner(ts, type_params, owner) == 0) { - ret = Py_None; + Py_INCREF(owner); + ret = owner; } done: Py_XDECREF(type_params); diff --git a/Python/codegen.c b/Python/codegen.c index fd3235e4e2dce4..19e1451d5576cb 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1594,10 +1594,7 @@ codegen_class(compiler *c, stmt_ty s) s->v.ClassDef.bases, &_Py_STR(generic_base), s->v.ClassDef.keywords)); - - ADDOP_I(c, loc, COPY, 1); ADDOP_I_IN_SCOPE(c, loc, CALL_INTRINSIC_1, INTRINSIC_SET_TYPE_PARAMS_OWNER); - ADDOP(c, loc, POP_TOP); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); From 1cdf1d7331aa2de506bddd448c1f48edd278c5c8 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Fri, 31 Jan 2025 15:14:22 -0500 Subject: [PATCH 06/12] set typevar owner explicitly one by one --- Doc/library/dis.rst | 83 ++++++++++++------------- Include/internal/pycore_compile.h | 1 + Include/internal/pycore_intrinsics.h | 6 +- Include/internal/pycore_typevarobject.h | 2 +- Objects/funcobject.c | 18 ++---- Objects/typevarobject.c | 82 ++++++------------------ Python/codegen.c | 25 ++++++-- Python/compile.c | 45 +++++++++----- Python/intrinsics.c | 2 +- 9 files changed, 120 insertions(+), 144 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 5eafda817a5229..f8f4188d27b472 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1778,50 +1778,45 @@ iterations of the loop. The operand determines which intrinsic function is called: - +-------------------------------------+-----------------------------------+ - | Operand | Description | - +=====================================+===================================+ - | ``INTRINSIC_1_INVALID`` | Not valid | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_PRINT`` | Prints the argument to standard | - | | out. Used in the REPL. | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_IMPORT_STAR`` | Performs ``import *`` for the | - | | named module. | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_STOPITERATION_ERROR`` | Extracts the return value from a | - | | ``StopIteration`` exception. | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an async generator value | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_UNARY_POSITIVE`` | Performs the unary ``+`` | - | | operation | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_LIST_TO_TUPLE`` | Converts a list to a tuple | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_TYPEVAR`` | Creates a :class:`typing.TypeVar` | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_PARAMSPEC`` | Creates a | - | | :class:`typing.ParamSpec` | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_TYPEVARTUPLE`` | Creates a | - | | :class:`typing.TypeVarTuple` | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_SUBSCRIPT_GENERIC`` | Returns :class:`typing.Generic` | - | | subscripted with the argument | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_TYPEALIAS`` | Creates a | - | | :class:`typing.TypeAliasType`; | - | | used in the :keyword:`type` | - | | statement. The argument is a tuple| - | | of the type alias's name, | - | | type parameters, and value. | - +-------------------------------------+-----------------------------------+ - | ``INTRINSIC_SET_TYPE_PARAMS_OWNER`` | Sets owner for any generics | - | | stored in an object's | - | | ``__type_params__`` to be that | - | | object. | - +-------------------------------------+-----------------------------------+ + +-----------------------------------+-----------------------------------+ + | Operand | Description | + +===================================+===================================+ + | ``INTRINSIC_1_INVALID`` | Not valid | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_PRINT`` | Prints the argument to standard | + | | out. Used in the REPL. | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_IMPORT_STAR`` | Performs ``import *`` for the | + | | named module. | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_STOPITERATION_ERROR`` | Extracts the return value from a | + | | ``StopIteration`` exception. | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_ASYNC_GEN_WRAP`` | Wraps an async generator value | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_UNARY_POSITIVE`` | Performs the unary ``+`` | + | | operation | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_LIST_TO_TUPLE`` | Converts a list to a tuple | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVAR`` | Creates a :class:`typing.TypeVar` | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_PARAMSPEC`` | Creates a | + | | :class:`typing.ParamSpec` | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVARTUPLE`` | Creates a | + | | :class:`typing.TypeVarTuple` | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_SUBSCRIPT_GENERIC`` | Returns :class:`typing.Generic` | + | | subscripted with the argument | + +-----------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEALIAS`` | Creates a | + | | :class:`typing.TypeAliasType`; | + | | used in the :keyword:`type` | + | | statement. The argument is a tuple| + | | of the type alias's name, | + | | type parameters, and value. | + +-----------------------------------+-----------------------------------+ .. versionadded:: 3.12 diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 9f0ca33892a43b..90fad8fad9df36 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -137,6 +137,7 @@ int _PyCompile_ScopeType(struct _PyCompiler *c); int _PyCompile_OptimizationLevel(struct _PyCompiler *c); int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name); PyObject *_PyCompile_Qualname(struct _PyCompiler *c); +PyObject *_PyCompile_PeekQualname(struct _PyCompiler *c, PyObject *name); _PyCompile_CodeUnitMetadata *_PyCompile_Metadata(struct _PyCompiler *c); PyObject *_PyCompile_StaticAttributesAsTuple(struct _PyCompiler *c); diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 983c403d2c51f6..bc7b78b3ba1dd9 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -18,9 +18,8 @@ #define INTRINSIC_TYPEVARTUPLE 9 #define INTRINSIC_SUBSCRIPT_GENERIC 10 #define INTRINSIC_TYPEALIAS 11 -#define INTRINSIC_SET_TYPE_PARAMS_OWNER 12 -#define MAX_INTRINSIC_1 12 +#define MAX_INTRINSIC_1 11 /* Binary Functions: */ @@ -30,8 +29,9 @@ #define INTRINSIC_TYPEVAR_WITH_CONSTRAINTS 3 #define INTRINSIC_SET_FUNCTION_TYPE_PARAMS 4 #define INTRINSIC_SET_TYPEPARAM_DEFAULT 5 +#define INTRINSIC_SET_TYPEPARAM_OWNER 6 -#define MAX_INTRINSIC_2 5 +#define MAX_INTRINSIC_2 6 typedef PyObject *(*intrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*intrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index dbf770f8bb29bc..ad2b985d7564c2 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -14,7 +14,7 @@ extern PyObject *_Py_make_typevartuple(PyThreadState *, PyObject *); extern PyObject *_Py_make_typealias(PyThreadState *, PyObject *); extern PyObject *_Py_subscript_generic(PyThreadState *, PyObject *); extern PyObject *_Py_set_typeparam_default(PyThreadState *, PyObject *, PyObject *); -extern PyObject *_Py_set_type_params_owner(PyThreadState *, PyObject *); +extern PyObject *_Py_set_typeparam_owner(PyThreadState *, PyObject *, PyObject *); extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); extern int _Py_typing_type_repr(PyUnicodeWriter *, PyObject *); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 668cf82b2f79de..7b17a9ba31fac4 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -2,12 +2,11 @@ /* Function object implementation */ #include "Python.h" -#include "pycore_dict.h" // _Py_INCREF_DICT() -#include "pycore_long.h" // _PyLong_GetOne() -#include "pycore_modsupport.h" // _PyArg_NoKeywords() -#include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_pyerrors.h" // _PyErr_Occurred() -#include "pycore_typevarobject.h" // _Py_set_type_params_owner() +#include "pycore_dict.h" // _Py_INCREF_DICT() +#include "pycore_long.h" // _PyLong_GetOne() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() +#include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_pyerrors.h" // _PyErr_Occurred() static const char * @@ -921,18 +920,13 @@ func_set_type_params(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) } PyObject * -_Py_set_function_type_params(PyThreadState *ts, PyObject *func, +_Py_set_function_type_params(PyThreadState *Py_UNUSED(ignored), PyObject *func, PyObject *type_params) { assert(PyFunction_Check(func)); assert(PyTuple_Check(type_params)); PyFunctionObject *f = (PyFunctionObject *)func; Py_XSETREF(f->func_typeparams, Py_NewRef(type_params)); - PyObject *res = _Py_set_type_params_owner(ts, func); - if (res == NULL) { - return NULL; - } - Py_DECREF(res); return Py_NewRef(func); } diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 5446eba5011d40..e5efcbc3188a1d 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -2451,8 +2451,8 @@ set_typeparam_owner(PyThreadState *ts, PyObject *typeparam, PyObject *owner) { if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { #ifdef Py_GIL_DISABLED - owner = _Py_atomic_exchange_ptr(&((typevarobject *)typeparam)->owner, owner); - assert(owner == NULL); + PyObject *oldowner = _Py_atomic_exchange_ptr(&((typevarobject *)typeparam)->owner, owner); + assert(oldowner == NULL); #else assert(((typevarobject *)typeparam)->owner == NULL); ((typevarobject *)typeparam)->owner = owner; @@ -2460,8 +2460,8 @@ set_typeparam_owner(PyThreadState *ts, PyObject *typeparam, PyObject *owner) } else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { #ifdef Py_GIL_DISABLED - owner = _Py_atomic_exchange_ptr(&((paramspecobject *)typeparam)->owner, owner); - assert(owner == NULL); + PyObject *oldowner = _Py_atomic_exchange_ptr(&((paramspecobject *)typeparam)->owner, owner); + assert(oldowner == NULL); #else assert(((paramspecobject *)typeparam)->owner == NULL); ((paramspecobject *)typeparam)->owner = owner; @@ -2469,8 +2469,8 @@ set_typeparam_owner(PyThreadState *ts, PyObject *typeparam, PyObject *owner) } else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { #ifdef Py_GIL_DISABLED - owner = _Py_atomic_exchange_ptr(&((typevartupleobject *)typeparam)->owner, owner); - assert(owner == NULL); + PyObject *oldowner = _Py_atomic_exchange_ptr(&((typevartupleobject *)typeparam)->owner, owner); + assert(oldowner == NULL); #else assert(((typevartupleobject *)typeparam)->owner == NULL); ((typevartupleobject *)typeparam)->owner = owner; @@ -2479,67 +2479,21 @@ set_typeparam_owner(PyThreadState *ts, PyObject *typeparam, PyObject *owner) else { return -1; } + Py_INCREF(owner); return 0; } -static int -set_type_params_owner(PyThreadState *ts, PyObject *type_params, PyObject *owner) -{ - assert(PyTuple_Check(type_params)); - int ret = -1; - PyObject *module = NULL; - PyObject *qualname = NULL; - - int res = PyObject_GetOptionalAttr(owner, &_Py_ID(__module__), &module); - if (res <= 0) { - goto done; - } - res = PyObject_GetOptionalAttr(owner, &_Py_ID(__qualname__), &qualname); - if (res <= 0) { - goto done; - } - - for (Py_ssize_t i = 0; i < Py_SIZE(type_params); i++) { - PyObject *index = PyLong_FromSsize_t(i); - if (index == NULL) { - goto done; - } - PyObject *owner_tuple = PyTuple_Pack(3, module, qualname, index); - Py_DECREF(index); - if (owner_tuple == NULL) { - goto done; - } - if (set_typeparam_owner(ts, PyTuple_GET_ITEM(type_params, i), owner_tuple) < 0) { - Py_DECREF(owner_tuple); - PyErr_SetString(PyExc_RuntimeError, "failed to set typeparam owner"); - goto done; - } - } - ret = 0; - -done: - Py_XDECREF(qualname); - Py_XDECREF(module); - return ret; -} - PyObject * -_Py_set_type_params_owner(PyThreadState *ts, PyObject *owner) -{ - PyObject *ret = NULL; - PyObject *type_params = PyObject_GetAttr(owner, &_Py_ID(__type_params__)); - if (type_params == NULL) { - goto done; - } - if (!PyTuple_Check(type_params)) { - PyErr_SetString(PyExc_ValueError, "expecting __type_params__ to be a tuple"); - goto done; - } - if (set_type_params_owner(ts, type_params, owner) == 0) { - Py_INCREF(owner); - ret = owner; +_Py_set_typeparam_owner(PyThreadState *ts, PyObject *typeparam, PyObject *owner) +{ + assert(PyTuple_CheckExact(owner) && PyTuple_GET_SIZE(owner) == 3); + assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(owner, 0))); + assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(owner, 1))); + assert(PyLong_CheckExact(PyTuple_GET_ITEM(owner, 2))); + if (set_typeparam_owner(ts, typeparam, owner) < 0) { + PyErr_SetString(PyExc_RuntimeError, "invalid typeparam"); + return NULL; } -done: - Py_XDECREF(type_params); - return ret; + Py_INCREF(typeparam); + return typeparam; } diff --git a/Python/codegen.c b/Python/codegen.c index 19e1451d5576cb..40f19588a91a25 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1147,13 +1147,23 @@ codegen_type_param_bound_or_default(compiler *c, expr_ty e, } static int -codegen_type_params(compiler *c, asdl_type_param_seq *type_params) +codegen_type_params(compiler *c, asdl_type_param_seq *type_params, PyObject *name) { if (!type_params) { return SUCCESS; } Py_ssize_t n = asdl_seq_LEN(type_params); bool seen_default = false; + PyObject *qualname = _PyCompile_PeekQualname(c, name); + if (qualname == NULL) { + return ERROR; + } + _PyCompile_optype modoptype; + Py_ssize_t modarg; + if (_PyCompile_ResolveNameop(c, &_Py_ID(__name__), GLOBAL_EXPLICIT, &modoptype, &modarg) < 0) { + return ERROR; + } + modarg <<= 1; for (Py_ssize_t i = 0; i < n; i++) { type_param_ty typeparam = asdl_seq_GET(type_params, i); @@ -1230,8 +1240,14 @@ codegen_type_params(compiler *c, asdl_type_param_seq *type_params) RETURN_IF_ERROR(codegen_nameop(c, loc, typeparam->v.ParamSpec.name, Store)); break; } + ADDOP_I(c, loc, LOAD_GLOBAL, modarg); + ADDOP_LOAD_CONST(c, loc, qualname); + ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromSsize_t(i)); + ADDOP_I(c, loc, BUILD_TUPLE, 3); + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_OWNER); } ADDOP_I(c, LOC(asdl_seq_GET(type_params, 0)), BUILD_TUPLE, n); + Py_DECREF(qualname); return SUCCESS; } @@ -1380,7 +1396,7 @@ codegen_function(compiler *c, stmt_ty s, int is_async) (void *)type_params, firstlineno, NULL, &umd); Py_DECREF(type_params_name); RETURN_IF_ERROR(ret); - RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params, name)); for (int i = 0; i < num_typeparam_args; i++) { ADDOP_I_IN_SCOPE(c, loc, LOAD_FAST, i); } @@ -1568,7 +1584,7 @@ codegen_class(compiler *c, stmt_ty s) (void *)type_params, firstlineno, s->v.ClassDef.name, NULL); Py_DECREF(type_params_name); RETURN_IF_ERROR(ret); - RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params, s->v.ClassDef.name)); _Py_DECLARE_STR(type_params, ".type_params"); RETURN_IF_ERROR_IN_SCOPE(c, codegen_nameop(c, loc, &_Py_STR(type_params), Store)); } @@ -1594,7 +1610,6 @@ codegen_class(compiler *c, stmt_ty s) s->v.ClassDef.bases, &_Py_STR(generic_base), s->v.ClassDef.keywords)); - ADDOP_I_IN_SCOPE(c, loc, CALL_INTRINSIC_1, INTRINSIC_SET_TYPE_PARAMS_OWNER); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); @@ -1666,7 +1681,7 @@ codegen_typealias(compiler *c, stmt_ty s) Py_DECREF(type_params_name); RETURN_IF_ERROR(ret); ADDOP_LOAD_CONST_IN_SCOPE(c, loc, name); - RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_type_params(c, type_params, name)); } else { ADDOP_LOAD_CONST(c, loc, name); diff --git a/Python/compile.c b/Python/compile.c index ef470830336dde..45802aa08641cf 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -213,8 +213,8 @@ _PyCompile_MaybeAddStaticAttributeToClass(compiler *c, expr_ty e) return SUCCESS; } -static int -compiler_set_qualname(compiler *c) +static PyObject * +compiler_calc_qualname(compiler *c, PyObject *u_name) { Py_ssize_t stack_size; struct compiler_unit *u = c->u; @@ -237,8 +237,7 @@ compiler_set_qualname(compiler *c) if (stack_size == 2) { // If we're immediately within the module, we can skip // the rest and just set the qualname to be the same as name. - u->u_metadata.u_qualname = Py_NewRef(u->u_metadata.u_name); - return SUCCESS; + return Py_NewRef(u_name); } capsule = PyList_GET_ITEM(c->c_stack, stack_size - 2); parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME); @@ -248,15 +247,17 @@ compiler_set_qualname(compiler *c) if (u->u_scope_type == COMPILE_SCOPE_FUNCTION || u->u_scope_type == COMPILE_SCOPE_ASYNC_FUNCTION || u->u_scope_type == COMPILE_SCOPE_CLASS) { - assert(u->u_metadata.u_name); - mangled = _Py_Mangle(parent->u_private, u->u_metadata.u_name); + assert(u_name); + mangled = _Py_Mangle(parent->u_private, u_name); if (!mangled) { - return ERROR; + return NULL; } scope = _PyST_GetScope(parent->u_ste, mangled); Py_DECREF(mangled); - RETURN_IF_ERROR(scope); + if (scope == -1) { + return NULL; + } assert(scope != GLOBAL_IMPLICIT); if (scope == GLOBAL_EXPLICIT) force_global = 1; @@ -271,7 +272,7 @@ compiler_set_qualname(compiler *c) base = PyUnicode_Concat(parent->u_metadata.u_qualname, &_Py_STR(dot_locals)); if (base == NULL) { - return ERROR; + return NULL; } } else { @@ -284,18 +285,28 @@ compiler_set_qualname(compiler *c) name = PyUnicode_Concat(base, _Py_LATIN1_CHR('.')); Py_DECREF(base); if (name == NULL) { - return ERROR; + return NULL; } - PyUnicode_Append(&name, u->u_metadata.u_name); + PyUnicode_Append(&name, u_name); if (name == NULL) { - return ERROR; + return NULL; } } else { - name = Py_NewRef(u->u_metadata.u_name); + name = Py_NewRef(u_name); } - u->u_metadata.u_qualname = name; + return name; +} + +static int +compiler_set_qualname(compiler *c) +{ + PyObject *qualname = compiler_calc_qualname(c, c->u->u_metadata.u_name); + if (qualname == NULL) { + return ERROR; + } + c->u->u_metadata.u_qualname = qualname; return SUCCESS; } @@ -1220,6 +1231,12 @@ _PyCompile_Qualname(compiler *c) return c->u->u_metadata.u_qualname; } +PyObject * +_PyCompile_PeekQualname(compiler *c, PyObject *name) +{ + return compiler_calc_qualname(c, name); +} + _PyCompile_CodeUnitMetadata * _PyCompile_Metadata(compiler *c) { diff --git a/Python/intrinsics.c b/Python/intrinsics.c index a98a342c7742d2..cef8c0a45b73c8 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -220,7 +220,6 @@ _PyIntrinsics_UnaryFunctions[] = { INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVARTUPLE, _Py_make_typevartuple) INTRINSIC_FUNC_ENTRY(INTRINSIC_SUBSCRIPT_GENERIC, _Py_subscript_generic) INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEALIAS, _Py_make_typealias) - INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_TYPE_PARAMS_OWNER, _Py_set_type_params_owner) }; @@ -264,6 +263,7 @@ _PyIntrinsics_BinaryFunctions[] = { INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVAR_WITH_CONSTRAINTS, make_typevar_with_constraints) INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_FUNCTION_TYPE_PARAMS, _Py_set_function_type_params) INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_TYPEPARAM_DEFAULT, _Py_set_typeparam_default) + INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_TYPEPARAM_OWNER, _Py_set_typeparam_owner) }; #undef INTRINSIC_FUNC_ENTRY From 0c2a469856f50aee343ef011cdc481c34ebe3cf5 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 6 Feb 2025 07:27:03 -0500 Subject: [PATCH 07/12] LOAD_GLOBAL __name__ properly using nameop --- Python/codegen.c | 8 +------- Python/symtable.c | 3 +++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index e6f0c74cbb4091..38ad73c0c6fc30 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1155,12 +1155,6 @@ codegen_type_params(compiler *c, asdl_type_param_seq *type_params, PyObject *nam if (qualname == NULL) { return ERROR; } - _PyCompile_optype modoptype; - Py_ssize_t modarg; - if (_PyCompile_ResolveNameop(c, &_Py_ID(__name__), GLOBAL_EXPLICIT, &modoptype, &modarg) < 0) { - return ERROR; - } - modarg <<= 1; for (Py_ssize_t i = 0; i < n; i++) { type_param_ty typeparam = asdl_seq_GET(type_params, i); @@ -1237,7 +1231,7 @@ codegen_type_params(compiler *c, asdl_type_param_seq *type_params, PyObject *nam RETURN_IF_ERROR(codegen_nameop(c, loc, typeparam->v.ParamSpec.name, Store)); break; } - ADDOP_I(c, loc, LOAD_GLOBAL, modarg); + RETURN_IF_ERROR(codegen_nameop(c, loc, &_Py_ID(__name__), Load)); ADDOP_LOAD_CONST(c, loc, qualname); ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromSsize_t(i)); ADDOP_I(c, loc, BUILD_TUPLE, 3); diff --git a/Python/symtable.c b/Python/symtable.c index 49bd01ba68ac9e..50b9a6c2e58409 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1678,6 +1678,9 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, return 0; } } + if (!symtable_add_def(st, &_Py_ID(__name__), DEF_GLOBAL, loc)) { + return 0; + } return 1; } From f143e984623083d35c3934a2efd5d4e74b2b3b24 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 1 Mar 2025 07:59:57 -0500 Subject: [PATCH 08/12] requested misc changes --- Lib/test/test_type_params.py | 9 +++++++++ Objects/typevarobject.c | 12 +++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 963f996d9d2020..5e24ad0d395e06 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1191,6 +1191,8 @@ class Class2[X, Y]: ... class Class3[X, *Y, **Z]: ... class Class4[X: int, Y: (bytes, str)]: ... class Class5(Generic[T]): pass +class Class6: + def meth[Baz](): return Class5[Baz]() class TypeParamsPickleTest(unittest.TestCase): @@ -1269,6 +1271,13 @@ def test_pickling_anonymous_typeparams(self): unpickled = pickle.loads(pickled) self.assertIs(unpickled, thing) + thing = Class6.meth() + pickled = pickle.dumps(thing) + unpickled = pickle.loads(pickled) + self.assertIs(unpickled.__orig_class__, thing.__orig_class__) + self.assertIs(unpickled.__orig_class__.__args__[0], + Class6.meth.__type_params__[0]) + class TypeParamsWeakRefTest(unittest.TestCase): def test_weakrefs(self): diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 2b72493bbdc93e..1aa80e7734c1c9 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -464,7 +464,8 @@ unpack_typevartuples(PyObject *params) static PyObject * typeparam_reduce_anonymous(PyObject *self, PyObject *owner) { - assert(PyTuple_Check(owner) && PyTuple_GET_SIZE(owner) == 3); + assert(PyTuple_Check(owner)); + assert(PyTuple_GET_SIZE(owner) == 3); PyObject *module_name = PyTuple_GET_ITEM(owner, 0); PyObject *qualname = PyTuple_GET_ITEM(owner, 1); PyObject *index = PyTuple_GET_ITEM(owner, 2); @@ -516,9 +517,6 @@ typeparam_reduce_anonymous(PyObject *self, PyObject *owner) goto done; } ret = PyTuple_Pack(2, restore_func, args); - if (ret == NULL) { - goto done; - } done: Py_XDECREF(args); @@ -2511,7 +2509,8 @@ set_typeparam_owner(PyThreadState *ts, PyObject *typeparam, PyObject *owner) PyObject * _Py_set_typeparam_owner(PyThreadState *ts, PyObject *typeparam, PyObject *owner) { - assert(PyTuple_CheckExact(owner) && PyTuple_GET_SIZE(owner) == 3); + assert(PyTuple_CheckExact(owner)); + assert(PyTuple_GET_SIZE(owner) == 3); assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(owner, 0))); assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(owner, 1))); assert(PyLong_CheckExact(PyTuple_GET_ITEM(owner, 2))); @@ -2519,6 +2518,5 @@ _Py_set_typeparam_owner(PyThreadState *ts, PyObject *typeparam, PyObject *owner) PyErr_SetString(PyExc_RuntimeError, "invalid typeparam"); return NULL; } - Py_INCREF(typeparam); - return typeparam; + return Py_NewRef(typeparam); } From 840023e4634d95e9009e9beb4cf8f42e73c81156 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 1 Mar 2025 08:20:41 -0500 Subject: [PATCH 09/12] move _restore_anonymous_typeparam into _typing --- Lib/typing.py | 4 ---- Modules/_typingmodule.c | 31 ++++++++++++++++++++++++++++ Modules/clinic/_typingmodule.c.h | 35 +++++++++++++++++++++++++++++++- Objects/typevarobject.c | 2 +- 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index bb92b144e401b8..66570db7a5bd74 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -502,10 +502,6 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f return t -def _restore_anonymous_typeparam(owner, index): - return owner.__type_params__[index] - - class _Final: """Mixin to prohibit subclassing.""" diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 09fbb3c5e8b91d..4b8446e4ad8d77 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -34,8 +34,39 @@ _typing__idfunc(PyObject *module, PyObject *x) } +/*[clinic input] +_typing._restore_anonymous_typeparam -> object + + owner: object + index: object + / + +Restore previously pickled anonymous type param from object.__type_params__. +[clinic start generated code]*/ + +static PyObject * +_typing__restore_anonymous_typeparam_impl(PyObject *module, PyObject *owner, + PyObject *index) +/*[clinic end generated code: output=00baec27dbf8d2d9 input=2f048db28d8124fb]*/ +{ + PyObject *ret = NULL; + PyObject *type_params = PyObject_GetAttr(owner, &_Py_ID(__type_params__)); + + if (type_params == NULL) { + goto done; + } + + ret = PyObject_GetItem(type_params, index); + +done: + Py_XDECREF(type_params); + return ret; +} + + static PyMethodDef typing_methods[] = { _TYPING__IDFUNC_METHODDEF + _TYPING__RESTORE_ANONYMOUS_TYPEPARAM_METHODDEF {NULL, NULL, 0, NULL} }; diff --git a/Modules/clinic/_typingmodule.c.h b/Modules/clinic/_typingmodule.c.h index ea415e67153ed8..b05b82face0298 100644 --- a/Modules/clinic/_typingmodule.c.h +++ b/Modules/clinic/_typingmodule.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(_typing__idfunc__doc__, "_idfunc($module, x, /)\n" "--\n" @@ -9,4 +11,35 @@ PyDoc_STRVAR(_typing__idfunc__doc__, #define _TYPING__IDFUNC_METHODDEF \ {"_idfunc", (PyCFunction)_typing__idfunc, METH_O, _typing__idfunc__doc__}, -/*[clinic end generated code: output=e7ea2a3cb7ab301a input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_typing__restore_anonymous_typeparam__doc__, +"_restore_anonymous_typeparam($module, owner, index, /)\n" +"--\n" +"\n" +"Restore previously pickled anonymous type param from object.__type_params__."); + +#define _TYPING__RESTORE_ANONYMOUS_TYPEPARAM_METHODDEF \ + {"_restore_anonymous_typeparam", _PyCFunction_CAST(_typing__restore_anonymous_typeparam), METH_FASTCALL, _typing__restore_anonymous_typeparam__doc__}, + +static PyObject * +_typing__restore_anonymous_typeparam_impl(PyObject *module, PyObject *owner, + PyObject *index); + +static PyObject * +_typing__restore_anonymous_typeparam(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *owner; + PyObject *index; + + if (!_PyArg_CheckPositional("_restore_anonymous_typeparam", nargs, 2, 2)) { + goto exit; + } + owner = args[0]; + index = args[1]; + return_value = _typing__restore_anonymous_typeparam_impl(module, owner, index); + +exit: + return return_value; +} +/*[clinic end generated code: output=ad8652d1dc62a084 input=a9049054013a1b77]*/ diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 1aa80e7734c1c9..3ec43fa19a3879 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -480,7 +480,7 @@ typeparam_reduce_anonymous(PyObject *self, PyObject *owner) PyObject *obj = NULL; PyObject *args = NULL; - typing = PyImport_ImportModule("typing"); + typing = PyImport_ImportModule("_typing"); if (typing == NULL) { goto done; } From 66ac0405fe89a8ce483ee898cd9f291f332639c5 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 1 Mar 2025 10:15:23 -0500 Subject: [PATCH 10/12] requested changes --- Lib/test/test_type_params.py | 11 +++++------ .../2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst | 2 +- Modules/_typingmodule.c | 13 ++++--------- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 5e24ad0d395e06..f34e84cba5cd4c 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1184,15 +1184,14 @@ def func2[X, Y](x: X | Y) -> X | Y: ... def func3[X, *Y, **Z](x: X, y: tuple[*Y], z: Z) -> X: ... def func4[X: int, Y: (bytes, str)](x: X, y: Y) -> X | Y: ... def func3b[X, *Y, **Z]() -> X: return Class3[X, Y, Z]() -def func5[Baz](): return Class5[Baz]() +def func5[Baz](): return Class1[Baz]() class Class1[X]: ... class Class2[X, Y]: ... class Class3[X, *Y, **Z]: ... class Class4[X: int, Y: (bytes, str)]: ... -class Class5(Generic[T]): pass -class Class6: - def meth[Baz](): return Class5[Baz]() +class Class5: + def meth[Baz](): return Class1[Baz]() class TypeParamsPickleTest(unittest.TestCase): @@ -1271,12 +1270,12 @@ def test_pickling_anonymous_typeparams(self): unpickled = pickle.loads(pickled) self.assertIs(unpickled, thing) - thing = Class6.meth() + thing = Class5.meth() pickled = pickle.dumps(thing) unpickled = pickle.loads(pickled) self.assertIs(unpickled.__orig_class__, thing.__orig_class__) self.assertIs(unpickled.__orig_class__.__args__[0], - Class6.meth.__type_params__[0]) + Class5.meth.__type_params__[0]) class TypeParamsWeakRefTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst index 490fe1c35c150c..206b0ab249ad23 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-29-15-32-02.gh-issue-129250.ExhmQQ.rst @@ -1 +1 @@ -Allow pickle of instances of generic classes. +Fix pickling of generic classes instances. diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 4b8446e4ad8d77..d6e16077a4d5df 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -49,18 +49,13 @@ _typing__restore_anonymous_typeparam_impl(PyObject *module, PyObject *owner, PyObject *index) /*[clinic end generated code: output=00baec27dbf8d2d9 input=2f048db28d8124fb]*/ { - PyObject *ret = NULL; PyObject *type_params = PyObject_GetAttr(owner, &_Py_ID(__type_params__)); - if (type_params == NULL) { - goto done; + return NULL; } - - ret = PyObject_GetItem(type_params, index); - -done: - Py_XDECREF(type_params); - return ret; + PyObject *res = PyObject_GetItem(type_params, index); + Py_DECREF(type_params); + return res; } From 0824e2f57e8bd173193276a2ea4aea8ee567a5dc Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 1 Mar 2025 10:26:21 -0500 Subject: [PATCH 11/12] requested changes --- Objects/typevarobject.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 3ec43fa19a3879..d69527b79fc19b 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -464,14 +464,14 @@ unpack_typevartuples(PyObject *params) static PyObject * typeparam_reduce_anonymous(PyObject *self, PyObject *owner) { - assert(PyTuple_Check(owner)); + assert(PyTuple_CheckExact(owner)); assert(PyTuple_GET_SIZE(owner) == 3); PyObject *module_name = PyTuple_GET_ITEM(owner, 0); PyObject *qualname = PyTuple_GET_ITEM(owner, 1); PyObject *index = PyTuple_GET_ITEM(owner, 2); - assert(PyUnicode_Check(module_name)); - assert(PyUnicode_Check(qualname)); - assert(PyLong_Check(index)); + assert(PyUnicode_CheckExact(module_name)); + assert(PyUnicode_CheckExact(qualname)); + assert(PyLong_CheckExact(index)); PyObject *ret = NULL; PyObject *typing = NULL; PyObject *restore_func = NULL; From db3ba3a2d3dbe5e875df21d5b36a60b4d3924067 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Fri, 4 Apr 2025 07:43:56 -0400 Subject: [PATCH 12/12] fix merge missing Py_ID() #include --- Modules/_typingmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 4d2f709845420c..41c00e751bf0f6 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -8,7 +8,8 @@ #include "internal/pycore_interp.h" #include "internal/pycore_typevarobject.h" #include "internal/pycore_unionobject.h" // _PyUnion_Type -#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_runtime.h" // _Py_ID() #include "clinic/_typingmodule.c.h" /*[clinic input]