From ee7333c124cfcf1abb9eccbc4b52969f443fa59d Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 21 May 2024 13:50:16 +0300 Subject: [PATCH 01/70] Initial Implementation --- Lib/functools.py | 52 ++++++++++++- Modules/_functoolsmodule.c | 151 ++++++++++++++++++++++++++++++++++--- 2 files changed, 187 insertions(+), 16 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index a80e1a6c6a56ac..a92ca705c597ba 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -273,6 +273,11 @@ def reduce(function, sequence, initial=_initial_missing): ### partial() argument application ################################################################################ + +class Placeholder: + """placeholder for partial arguments""" + + # Purely functional, no descriptor behaviour class partial: """New function with partial application of the given arguments @@ -285,8 +290,33 @@ def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") + np = 0 + nargs = len(args) + if args: + while nargs and args[nargs-1] is Placeholder: + nargs -= 1 + args = args[:nargs] + np = args.count(Placeholder) if isinstance(func, partial): - args = func.args + args + pargs = func.args + pnp = func.np + if pnp and args: + all_args = list(pargs) + nargs = len(args) + j, pos = 0, 0 + end = nargs if nargs < pnp else pnp + while j < end: + pos = all_args.index(Placeholder, pos) + all_args[pos] = args[j] + j += 1 + pos += 1 + if pnp < nargs: + all_args.extend(args[pnp:]) + np += pnp - end + args = tuple(all_args) + else: + np += pnp + args = func.args + args keywords = {**func.keywords, **keywords} func = func.func @@ -295,11 +325,25 @@ def __new__(cls, func, /, *args, **keywords): self.func = func self.args = args self.keywords = keywords + self.np = np return self def __call__(self, /, *args, **keywords): - keywords = {**self.keywords, **keywords} - return self.func(*self.args, *args, **keywords) + if np := self.np: + if len(args) < np: + raise ValueError("unfilled placeholders in 'partial' call") + f_args = list(self.args) + j, pos = 0, 0 + while j < np: + pos = f_args.index(Placeholder, pos) + f_args[pos] = args[j] + j += 1 + pos += 1 + keywords = {**self.keywords, **keywords} + return self.func(*f_args, *args[np:], **keywords) + else: + keywords = {**self.keywords, **keywords} + return self.func(*self.args, *args, **keywords) @recursive_repr() def __repr__(self): @@ -340,7 +384,7 @@ def __setstate__(self, state): self.keywords = kwds try: - from _functools import partial + from _functools import partial, Placeholder except ImportError: pass diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 9dee7bf3062710..84e6269000b644 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -41,6 +41,23 @@ get_functools_state(PyObject *module) /* partial object **********************************************************/ + +typedef struct { + PyObject_HEAD +} placeholderobject; + + +static PyTypeObject placeholder_type = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "functools.Placeholder", + .tp_doc = PyDoc_STR("placeholder for partial arguments"), + .tp_basicsize = sizeof(placeholderobject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_new = PyType_GenericNew, +}; + + typedef struct { PyObject_HEAD PyObject *fn; @@ -48,6 +65,7 @@ typedef struct { PyObject *kw; PyObject *dict; /* __dict__ */ PyObject *weakreflist; /* List of weak references */ + Py_ssize_t np; /* Number of placeholders */ vectorcallfunc vectorcall; } partialobject; @@ -72,6 +90,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *func, *pargs, *nargs, *pkw; partialobject *pto; + Py_ssize_t pnp = 0; if (PyTuple_GET_SIZE(args) < 1) { PyErr_SetString(PyExc_TypeError, @@ -98,6 +117,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) pargs = part->args; pkw = part->kw; func = part->fn; + pnp = part->np; assert(PyTuple_Check(pargs)); assert(PyDict_Check(pkw)); } @@ -120,11 +140,57 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) Py_DECREF(pto); return NULL; } - if (pargs == NULL) { - pto->args = nargs; + Py_ssize_t nnp = 0; + Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); + PyObject *item; + if (nnargs > 0){ + Py_ssize_t i; + for (i=nnargs; i > 0; i--) { + item = PyTuple_GET_ITEM(nargs, i); + if (!Py_Is(item, (PyObject *) &placeholder_type)) + break; + } + if (i != nnargs) + nnargs = i; + + if (nnargs > 0){ + for (Py_ssize_t i=0; i < nnargs; i++){ + item = PyTuple_GET_ITEM(nargs, i); + nnp += Py_Is(item, (PyObject *) &placeholder_type); + } + } } - else { + if ((pnp > 0) & (nnargs > 0)) { + Py_ssize_t pnargs = PyTuple_GET_SIZE(pargs); + Py_ssize_t anargs = pnargs; + if (nnargs > pnp) + anargs += nnargs - pnp; + PyObject *aargs = PyTuple_New(anargs); + Py_ssize_t j = 0; + for (Py_ssize_t i=0; i < anargs; i++) { + if (i < pnargs) { + item = PyTuple_GET_ITEM(pargs, i); + if ((j < nnargs) & Py_Is(item, (PyObject *) &placeholder_type)){ + item = PyTuple_GET_ITEM(nargs, j); + j++; + pnp--; + } + } else { + item = PyTuple_GET_ITEM(nargs, j); + j++; + } + Py_INCREF(item); + PyTuple_SET_ITEM(aargs, i, item); + } + pto->args = aargs; + pto->np = pnp + nnp; + Py_DECREF(nargs); + } else if (pargs == NULL) { + pto->args = nargs; + pto->np = nnp; + } else { pto->args = PySequence_Concat(pargs, nargs); + pto->np = pnp + nnp; Py_DECREF(nargs); if (pto->args == NULL) { Py_DECREF(pto); @@ -217,13 +283,19 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, size_t nargsf, PyObject *kwnames) { PyThreadState *tstate = _PyThreadState_GET(); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); /* pto->kw is mutable, so need to check every time */ if (PyDict_GET_SIZE(pto->kw)) { return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames); } + Py_ssize_t np = pto->np; + if (nargs < np) { + PyErr_SetString(PyExc_TypeError, + "unfilled placeholders in 'partial' call"); + return NULL; + } - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); Py_ssize_t nargs_total = nargs; if (kwnames != NULL) { nargs_total += PyTuple_GET_SIZE(kwnames); @@ -251,6 +323,7 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, } Py_ssize_t newnargs_total = pto_nargs + nargs_total; + newnargs_total = newnargs_total - np; PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; PyObject *ret; @@ -267,12 +340,28 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, } } - /* Copy to new stack, using borrowed references */ - memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); - memcpy(stack + pto_nargs, args, nargs_total * sizeof(PyObject*)); - - ret = _PyObject_VectorcallTstate(tstate, pto->fn, - stack, pto_nargs + nargs, kwnames); + Py_ssize_t nargs_new; + if (np) { + nargs_new = pto_nargs + nargs - np; + Py_ssize_t j = 0; // Placeholder counter + for (Py_ssize_t i=0; i < pto_nargs; i++) { + if (Py_Is(pto_args[i], (PyObject *) &placeholder_type)){ + memcpy(stack + i, args + j, 1 * sizeof(PyObject*)); + j += 1; + } else { + memcpy(stack + i, pto_args + i, 1 * sizeof(PyObject*)); + } + } + if (nargs_total > np){ + memcpy(stack + pto_nargs, args + np, (nargs_total - np) * sizeof(PyObject*)); + } + } else { + nargs_new = pto_nargs + nargs; + /* Copy to new stack, using borrowed references */ + memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); + memcpy(stack + pto_nargs, args, nargs_total * sizeof(PyObject*)); + } + ret = _PyObject_VectorcallTstate(tstate, pto->fn, stack, nargs_new, kwnames); if (stack != small_stack) { PyMem_Free(stack); } @@ -304,6 +393,14 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) assert(PyTuple_Check(pto->args)); assert(PyDict_Check(pto->kw)); + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t np = pto->np; + if (nargs < np) { + PyErr_SetString(PyExc_TypeError, + "unfilled placeholders in 'partial' call"); + return NULL; + } + /* Merge keywords */ PyObject *kwargs2; if (PyDict_GET_SIZE(pto->kw) == 0) { @@ -328,8 +425,33 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) } /* Merge positional arguments */ - /* Note: tupleconcat() is optimized for empty tuples */ - PyObject *args2 = PySequence_Concat(pto->args, args); + PyObject *args2; + if (np) { + Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); + Py_ssize_t nargs_new = pto_nargs + nargs - np; + args2 = PyTuple_New(nargs_new); + PyObject *pto_args = pto->args; + PyObject *item; + Py_ssize_t j = 0; // Placeholder counter + for (Py_ssize_t i=0; i < pto_nargs; i++) { + item = PyTuple_GET_ITEM(pto_args, i); + if (Py_Is(item, (PyObject *) &placeholder_type)){ + item = PyTuple_GET_ITEM(args, j); + j += 1; + } + PyTuple_SET_ITEM(args2, i, item); + } + if (nargs > np){ + for (Py_ssize_t i=pto_nargs; i < nargs_new; i++) { + item = PyTuple_GET_ITEM(args, j); + PyTuple_SET_ITEM(args2, i, item); + j += 1; + } + } + } else { + /* Note: tupleconcat() is optimized for empty tuples */ + args2 = PySequence_Concat(pto->args, args); + } if (args2 == NULL) { Py_XDECREF(kwargs2); return NULL; @@ -354,6 +476,8 @@ static PyMemberDef partial_memberlist[] = { "tuple of arguments to future partial calls"}, {"keywords", _Py_T_OBJECT, OFF(kw), Py_READONLY, "dictionary of keyword arguments to future partial calls"}, + {"np", Py_T_PYSSIZET, OFF(np), Py_READONLY, + "number of placeholders"}, {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(partialobject, weakreflist), Py_READONLY}, {"__dictoffset__", Py_T_PYSSIZET, @@ -1497,6 +1621,9 @@ _functools_exec(PyObject *module) if (PyModule_AddType(module, state->partial_type) < 0) { return -1; } + if (PyModule_AddType(module, &placeholder_type) < 0) { + return -1; + } PyObject *lru_cache_type = PyType_FromModuleAndSpec(module, &lru_cache_type_spec, NULL); From 8bcc4623035c4f427fd9ff65b18aa7b3c034a656 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 21 May 2024 14:22:30 +0300 Subject: [PATCH 02/70] serialization fix --- Lib/functools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/functools.py b/Lib/functools.py index a92ca705c597ba..4dfa69f1d95319 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -329,6 +329,8 @@ def __new__(cls, func, /, *args, **keywords): return self def __call__(self, /, *args, **keywords): + if not hasattr(self, 'np'): + self.np = self.args.count(Placeholder) if np := self.np: if len(args) < np: raise ValueError("unfilled placeholders in 'partial' call") From c67c9b47bd0bf0d8b5aba4ba00d8c8d209fea20f Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 21 May 2024 17:08:19 +0300 Subject: [PATCH 03/70] bug fix --- Modules/_functoolsmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 84e6269000b644..efc58ca2b16b7b 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -156,7 +156,8 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (nnargs > 0){ for (Py_ssize_t i=0; i < nnargs; i++){ item = PyTuple_GET_ITEM(nargs, i); - nnp += Py_Is(item, (PyObject *) &placeholder_type); + if (Py_Is(item, (PyObject *) &placeholder_type)) + nnp++; } } } From 680d900d3823f311c30726541b8bd99ca3a56cb8 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 21 May 2024 17:43:47 +0300 Subject: [PATCH 04/70] Bug 2 fix --- Modules/_functoolsmodule.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index efc58ca2b16b7b..1a860109900f0d 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -145,10 +145,11 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) PyObject *item; if (nnargs > 0){ Py_ssize_t i; - for (i=nnargs; i > 0; i--) { - item = PyTuple_GET_ITEM(nargs, i); + for (i=nnargs; i > 0;) { + item = PyTuple_GET_ITEM(nargs, i-1); if (!Py_Is(item, (PyObject *) &placeholder_type)) break; + i--; } if (i != nnargs) nnargs = i; From 9591ff57b71f7e0341b62ec0d280582d5467c796 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 21 May 2024 18:26:39 +0300 Subject: [PATCH 05/70] Py_TPFLAGS_IMMUTABLETYPE added --- Modules/_functoolsmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 1a860109900f0d..9966c39947caa4 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -53,7 +53,8 @@ static PyTypeObject placeholder_type = { .tp_doc = PyDoc_STR("placeholder for partial arguments"), .tp_basicsize = sizeof(placeholderobject), .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_new = PyType_GenericNew, }; From 067e93834a477857c0cad11079d5996ece415139 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 23 May 2024 17:21:05 +0300 Subject: [PATCH 06/70] placeholder added to state as opposed to being used as global constant --- Modules/_functoolsmodule.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 9966c39947caa4..3f247e3c8a2820 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -25,6 +25,7 @@ class _functools._lru_cache_wrapper "PyObject *" "&lru_cache_type_spec" typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; + PyObject *placeholder; PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -66,6 +67,7 @@ typedef struct { PyObject *kw; PyObject *dict; /* __dict__ */ PyObject *weakreflist; /* List of weak references */ + PyObject *placeholder; Py_ssize_t np; /* Number of placeholders */ vectorcallfunc vectorcall; } partialobject; @@ -141,6 +143,8 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) Py_DECREF(pto); return NULL; } + + pto->placeholder = state->placeholder; Py_ssize_t nnp = 0; Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); PyObject *item; @@ -148,7 +152,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) Py_ssize_t i; for (i=nnargs; i > 0;) { item = PyTuple_GET_ITEM(nargs, i-1); - if (!Py_Is(item, (PyObject *) &placeholder_type)) + if (!Py_Is(item, pto->placeholder)) break; i--; } @@ -158,7 +162,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (nnargs > 0){ for (Py_ssize_t i=0; i < nnargs; i++){ item = PyTuple_GET_ITEM(nargs, i); - if (Py_Is(item, (PyObject *) &placeholder_type)) + if (Py_Is(item, pto->placeholder)) nnp++; } } @@ -173,7 +177,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) for (Py_ssize_t i=0; i < anargs; i++) { if (i < pnargs) { item = PyTuple_GET_ITEM(pargs, i); - if ((j < nnargs) & Py_Is(item, (PyObject *) &placeholder_type)){ + if ((j < nnargs) & Py_Is(item, pto->placeholder)){ item = PyTuple_GET_ITEM(nargs, j); j++; pnp--; @@ -348,7 +352,7 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, nargs_new = pto_nargs + nargs - np; Py_ssize_t j = 0; // Placeholder counter for (Py_ssize_t i=0; i < pto_nargs; i++) { - if (Py_Is(pto_args[i], (PyObject *) &placeholder_type)){ + if (Py_Is(pto_args[i], pto->placeholder)){ memcpy(stack + i, args + j, 1 * sizeof(PyObject*)); j += 1; } else { @@ -438,7 +442,7 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) Py_ssize_t j = 0; // Placeholder counter for (Py_ssize_t i=0; i < pto_nargs; i++) { item = PyTuple_GET_ITEM(pto_args, i); - if (Py_Is(item, (PyObject *) &placeholder_type)){ + if (Py_Is(item, pto->placeholder)){ item = PyTuple_GET_ITEM(args, j); j += 1; } @@ -1616,6 +1620,14 @@ _functools_exec(PyObject *module) return -1; } + state->placeholder = (PyObject *)&placeholder_type; + if (state->placeholder == NULL) { + return -1; + } + if (PyModule_AddType(module, &placeholder_type) < 0) { + return -1; + } + state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL); if (state->partial_type == NULL) { @@ -1624,9 +1636,6 @@ _functools_exec(PyObject *module) if (PyModule_AddType(module, state->partial_type) < 0) { return -1; } - if (PyModule_AddType(module, &placeholder_type) < 0) { - return -1; - } PyObject *lru_cache_type = PyType_FromModuleAndSpec(module, &lru_cache_type_spec, NULL); @@ -1663,6 +1672,7 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) { _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); + Py_VISIT(state->placeholder); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1674,6 +1684,7 @@ _functools_clear(PyObject *module) { _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); + Py_CLEAR(state->placeholder); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); From 8af20b3b94eff0351e4baec41aff8e2208645816 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 23 May 2024 17:44:46 +0300 Subject: [PATCH 07/70] static removed --- Modules/_functoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 3f247e3c8a2820..e1a9f01cd97c65 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -48,7 +48,7 @@ typedef struct { } placeholderobject; -static PyTypeObject placeholder_type = { +PyTypeObject placeholder_type = { .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "functools.Placeholder", .tp_doc = PyDoc_STR("placeholder for partial arguments"), From 607a0b1a90bb0a358357ffe102c26329d230d26b Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 23 May 2024 18:06:57 +0300 Subject: [PATCH 08/70] creating sentinel via PyType_Spec --- Modules/_functoolsmodule.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index e1a9f01cd97c65..b59aea634afc68 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -25,7 +25,7 @@ class _functools._lru_cache_wrapper "PyObject *" "&lru_cache_type_spec" typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; - PyObject *placeholder; + PyTypeObject *placeholder; PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -48,15 +48,21 @@ typedef struct { } placeholderobject; -PyTypeObject placeholder_type = { - .ob_base = PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "functools.Placeholder", - .tp_doc = PyDoc_STR("placeholder for partial arguments"), - .tp_basicsize = sizeof(placeholderobject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | - Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_new = PyType_GenericNew, +PyDoc_STRVAR(placeholder_doc, "placeholder for partial class"); + + +static PyType_Slot placeholder_type_slots[] = { + {Py_tp_doc, (void *)placeholder_doc}, + {0, 0} +}; + +static PyType_Spec placeholder_type_spec = { + .name = "partial2.Placeholder", + .basicsize = sizeof(placeholderobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION, + .slots = placeholder_type_slots }; @@ -144,7 +150,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - pto->placeholder = state->placeholder; + pto->placeholder = (PyObject *) state->placeholder; Py_ssize_t nnp = 0; Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); PyObject *item; @@ -1620,11 +1626,12 @@ _functools_exec(PyObject *module) return -1; } - state->placeholder = (PyObject *)&placeholder_type; + state->placeholder = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &placeholder_type_spec, NULL); if (state->placeholder == NULL) { return -1; } - if (PyModule_AddType(module, &placeholder_type) < 0) { + if (PyModule_AddType(module, state->placeholder) < 0) { return -1; } From f55801ede6ff4d86c33f3aef827c754abb69246c Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 23 May 2024 18:29:32 +0300 Subject: [PATCH 09/70] more accurate variable name --- Modules/_functoolsmodule.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index b59aea634afc68..0f733f5d1baa3e 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -25,7 +25,7 @@ class _functools._lru_cache_wrapper "PyObject *" "&lru_cache_type_spec" typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; - PyTypeObject *placeholder; + PyTypeObject *placeholder_type; PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -150,7 +150,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - pto->placeholder = (PyObject *) state->placeholder; + pto->placeholder = (PyObject *) state->placeholder_type; Py_ssize_t nnp = 0; Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); PyObject *item; @@ -1626,12 +1626,12 @@ _functools_exec(PyObject *module) return -1; } - state->placeholder = (PyTypeObject *)PyType_FromModuleAndSpec(module, + state->placeholder_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &placeholder_type_spec, NULL); - if (state->placeholder == NULL) { + if (state->placeholder_type == NULL) { return -1; } - if (PyModule_AddType(module, state->placeholder) < 0) { + if (PyModule_AddType(module, state->placeholder_type) < 0) { return -1; } @@ -1679,7 +1679,7 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) { _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); - Py_VISIT(state->placeholder); + Py_VISIT(state->placeholder_type); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1691,7 +1691,7 @@ _functools_clear(PyObject *module) { _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); - Py_CLEAR(state->placeholder); + Py_CLEAR(state->placeholder_type); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); From 5894145540f6b5c65cbfee85d42b69b4cc54ff62 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Fri, 24 May 2024 18:25:11 +0300 Subject: [PATCH 10/70] trailing trim bug and tests --- Lib/functools.py | 2 +- Lib/test/test_functools.py | 37 +++++++++++++++++++++++++++++++++++++ Modules/_functoolsmodule.c | 15 ++++++++++----- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 4dfa69f1d95319..d0016f8bdb9ba2 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -333,7 +333,7 @@ def __call__(self, /, *args, **keywords): self.np = self.args.count(Placeholder) if np := self.np: if len(args) < np: - raise ValueError("unfilled placeholders in 'partial' call") + raise TypeError("unfilled placeholders in 'partial' call") f_args = list(self.args) j, pos = 0, 0 while j < np: diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 4a9a7313712f60..4bcdd7c8b8c034 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -210,6 +210,43 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') + def test_placeholders_trailing_trim(self): + PH = self.module.Placeholder + for args in [(PH,), (PH, 0), (0, PH), (0, PH, 1, PH, PH, PH)]: + expected, n = tuple(args), len(args) + while n and args[n-1] is PH: + n -= 1 + expected = expected[:n] + p = self.partial(capture, *args) + self.assertTrue(p.args == expected) + + def test_placeholders(self): + PH = self.module.Placeholder + # 1 Placeholder + args = (PH, 0) + p = self.partial(capture, *args) + got, empty = p('x') + self.assertTrue(('x', 0) == got and empty == {}) + # 2 Placeholders + args = (PH, 0, PH, 1) + p = self.partial(capture, *args) + with self.assertRaises(TypeError): + got, empty = p('x') + got, empty = p('x', 'y') + expected = ('x', 0, 'y', 1) + self.assertTrue(expected == got and empty == {}) + + def test_placeholders_optimization(self): + PH = self.module.Placeholder + p = self.partial(capture, PH, 0) + p2 = self.partial(p, PH, 1, 2, 3) + expected = (PH, 0, 1, 2, 3) + self.assertTrue(expected == p2.args) + p3 = self.partial(p2, -1, 4) + got, empty = p3(5) + expected = (-1, 0, 1, 2, 3, 4, 5) + self.assertTrue(expected == got and empty == {}) + def test_repr(self): args = (object(), object()) args_repr = ', '.join(repr(a) for a in args) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 0f733f5d1baa3e..277c0234e5c832 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -48,7 +48,7 @@ typedef struct { } placeholderobject; -PyDoc_STRVAR(placeholder_doc, "placeholder for partial class"); +PyDoc_STRVAR(placeholder_doc, "placeholder for partial arguments"); static PyType_Slot placeholder_type_slots[] = { @@ -57,7 +57,7 @@ static PyType_Slot placeholder_type_slots[] = { }; static PyType_Spec placeholder_type_spec = { - .name = "partial2.Placeholder", + .name = "partial.Placeholder", .basicsize = sizeof(placeholderobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_IMMUTABLETYPE | @@ -73,7 +73,7 @@ typedef struct { PyObject *kw; PyObject *dict; /* __dict__ */ PyObject *weakreflist; /* List of weak references */ - PyObject *placeholder; + PyObject *placeholder; /* Placeholder for positional arguments */ Py_ssize_t np; /* Number of placeholders */ vectorcallfunc vectorcall; } partialobject; @@ -155,6 +155,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); PyObject *item; if (nnargs > 0){ + /* Trim placeholders from the end if needed */ Py_ssize_t i; for (i=nnargs; i > 0;) { item = PyTuple_GET_ITEM(nargs, i-1); @@ -162,9 +163,13 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) break; i--; } - if (i != nnargs) + if (i != nnargs){ nnargs = i; - + PyObject *tmp = PyTuple_GetSlice(nargs, 0, nnargs); + Py_DECREF(nargs); + nargs = tmp; + } + /* Count placeholders */ if (nnargs > 0){ for (Py_ssize_t i=0; i < nnargs; i++){ item = PyTuple_GET_ITEM(nargs, i); From 3722e073f67a3aeafa729cd06c6e51f466aa0ba3 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 25 May 2024 00:38:12 +0300 Subject: [PATCH 11/70] Updated docs --- Doc/library/functools.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 9d5c72802a21f2..73f1c07767a327 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -358,6 +358,18 @@ The :mod:`functools` module defines the following functions: >>> basetwo('10010') 18 + If :class:`Placeholder` sentinels are present in *args*, they will be filled first + when :func:`partial` is called. This allows custom selection of positional arguments + to be pre-filled when constructing :ref:`partial object`. + If :class:`Placeholder` sentinels are used, all of them must be filled at call time.: + + >>> from functools import partial, Placeholder + >>> say_to_world = partial(print, Placeholder, 'world!') + >>> say_to_world('Hello') + Hello world! + + .. versionchanged:: 3.14 + Support for :class:`Placeholder` in *args* .. class:: partialmethod(func, /, *args, **keywords) From a79c2af3fe6e2da02d7e8e77e8661b99639e1270 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 25 May 2024 00:55:26 +0300 Subject: [PATCH 12/70] blurb --- .../next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst b/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst new file mode 100644 index 00000000000000..70bb0ce98bb555 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst @@ -0,0 +1,2 @@ +``partial`` of ``functools`` module now supports placeholders for positional +arguments. From 92c767b9d02b8296b52a74e87d35a71e53f8c951 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 25 May 2024 02:35:13 +0300 Subject: [PATCH 13/70] minor edit --- Modules/_functoolsmodule.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 277c0234e5c832..65dbf760a0d0ed 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -157,11 +157,10 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (nnargs > 0){ /* Trim placeholders from the end if needed */ Py_ssize_t i; - for (i=nnargs; i > 0;) { + for (i=nnargs; i > 0; i--) { item = PyTuple_GET_ITEM(nargs, i-1); if (!Py_Is(item, pto->placeholder)) break; - i--; } if (i != nnargs){ nnargs = i; From 496a9d2caa7f867a806ee7574688fb9961975b20 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 25 May 2024 02:49:33 +0300 Subject: [PATCH 14/70] doc fix --- Doc/library/functools.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 73f1c07767a327..0e43d9333b614e 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -358,10 +358,10 @@ The :mod:`functools` module defines the following functions: >>> basetwo('10010') 18 - If :class:`Placeholder` sentinels are present in *args*, they will be filled first + If ``Placeholder`` sentinels are present in *args*, they will be filled first when :func:`partial` is called. This allows custom selection of positional arguments to be pre-filled when constructing :ref:`partial object`. - If :class:`Placeholder` sentinels are used, all of them must be filled at call time.: + If ``Placeholder`` sentinels are used, all of them must be filled at call time.: >>> from functools import partial, Placeholder >>> say_to_world = partial(print, Placeholder, 'world!') @@ -369,7 +369,7 @@ The :mod:`functools` module defines the following functions: Hello world! .. versionchanged:: 3.14 - Support for :class:`Placeholder` in *args* + Support for ``Placeholder`` in *args* .. class:: partialmethod(func, /, *args, **keywords) From 38d9c1185cbc6e27b8f1f1c348e87931d9df52c8 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Fri, 31 May 2024 05:55:37 +0300 Subject: [PATCH 15/70] better variable names and mini corrections --- Lib/_pyrepl/simple_interact.py | 4 ++-- Lib/test/test_typing.py | 2 +- Modules/_functoolsmodule.c | 32 +++++++++++++++----------------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 8ab4dab757685e..ffc1f275b2debe 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -97,8 +97,8 @@ def runsource(self, source, filename="", symbol="single"): try: code = compile(item, filename, the_symbol) except (OverflowError, ValueError): - self.showsyntaxerror(filename) - return False + self.showsyntaxerror(filename) + return False if code is None: return True diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index dac55ceb9e99e0..a9271079b9a789 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7345,7 +7345,7 @@ def test_no_generator_instantiation(self): def test_async_generator(self): async def f(): - yield 42 + yield 42 g = f() self.assertIsSubclass(type(g), typing.AsyncGenerator) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 65dbf760a0d0ed..055c123f61511f 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -156,36 +156,34 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) PyObject *item; if (nnargs > 0){ /* Trim placeholders from the end if needed */ - Py_ssize_t i; - for (i=nnargs; i > 0; i--) { - item = PyTuple_GET_ITEM(nargs, i-1); + Py_ssize_t nnargs_old = nnargs; + for (; nnargs > 0; nnargs--) { + item = PyTuple_GET_ITEM(nargs, nnargs-1); if (!Py_Is(item, pto->placeholder)) break; } - if (i != nnargs){ - nnargs = i; + if (nnargs != nnargs_old) { PyObject *tmp = PyTuple_GetSlice(nargs, 0, nnargs); Py_DECREF(nargs); nargs = tmp; } /* Count placeholders */ - if (nnargs > 0){ - for (Py_ssize_t i=0; i < nnargs; i++){ + if (nnargs > 1){ + for (Py_ssize_t i=0; i < nnargs - 1; i++){ item = PyTuple_GET_ITEM(nargs, i); if (Py_Is(item, pto->placeholder)) nnp++; } } } - if ((pnp > 0) & (nnargs > 0)) { - Py_ssize_t pnargs = PyTuple_GET_SIZE(pargs); - Py_ssize_t anargs = pnargs; + if ((pnp > 0) && (nnargs > 0)) { + Py_ssize_t npargs = PyTuple_GET_SIZE(pargs); + Py_ssize_t nfargs = npargs; if (nnargs > pnp) - anargs += nnargs - pnp; - PyObject *aargs = PyTuple_New(anargs); - Py_ssize_t j = 0; - for (Py_ssize_t i=0; i < anargs; i++) { - if (i < pnargs) { + nfargs += nnargs - pnp; + PyObject *fargs = PyTuple_New(nfargs); + for (Py_ssize_t i=0, j=0; i < nfargs; i++) { + if (i < npargs) { item = PyTuple_GET_ITEM(pargs, i); if ((j < nnargs) & Py_Is(item, pto->placeholder)){ item = PyTuple_GET_ITEM(nargs, j); @@ -197,9 +195,9 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) j++; } Py_INCREF(item); - PyTuple_SET_ITEM(aargs, i, item); + PyTuple_SET_ITEM(fargs, i, item); } - pto->args = aargs; + pto->args = fargs; pto->np = pnp + nnp; Py_DECREF(nargs); } else if (pargs == NULL) { From 14b38ca88b08def7d8a4733a8818632330d2da7c Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 8 Jun 2024 09:19:27 +0300 Subject: [PATCH 16/70] review comments mostly --- Doc/library/functools.rst | 24 +++++++++++++- Lib/functools.py | 45 +++++++++++++------------ Modules/_functoolsmodule.c | 68 +++++++++++++++++++++++--------------- 3 files changed, 88 insertions(+), 49 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 0e43d9333b614e..47f3d13f560489 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -337,13 +337,33 @@ The :mod:`functools` module defines the following functions: supplied, they extend and override *keywords*. Roughly equivalent to:: + Placeholder = object() + def partial(func, /, *args, **keywords): + # Trim trailing placeholders and count how many remaining + nargs = len(args) + while nargs and args[nargs-1] is Placeholder: + nargs -= 1 + args = args[:nargs] + placeholder_count = args.count(Placeholder) def newfunc(*fargs, **fkeywords): + if len(fargs) < placeholder_count: + raise TypeError("missing positional arguments in 'partial' call;" + f" expected at least {placeholder_count}, " + f"got {len(fargs)}") + newargs = list(args) + j = 0 + for i in range(len(args)): + if args[i] is Placeholder: + newargs[i] = fargs[j] + j += 1 + newargs.extend(fargs[j:]) newkeywords = {**keywords, **fkeywords} - return func(*args, *fargs, **newkeywords) + return func(*newargs, **newkeywords) newfunc.func = func newfunc.args = args newfunc.keywords = keywords + newfunc.placeholder_count = placeholder_count return newfunc The :func:`partial` is used for partial function application which "freezes" @@ -365,6 +385,8 @@ The :mod:`functools` module defines the following functions: >>> from functools import partial, Placeholder >>> say_to_world = partial(print, Placeholder, 'world!') + >>> say_to_world.placeholder_count + 1 >>> say_to_world('Hello') Hello world! diff --git a/Lib/functools.py b/Lib/functools.py index d0016f8bdb9ba2..7bcee5fdcf76cc 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -284,22 +284,22 @@ class partial: and keywords. """ - __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" + __slots__ = ("func", "args", "keywords", "placeholder_count", + "__dict__", "__weakref__") def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") - - np = 0 + placeholder_count = 0 nargs = len(args) if args: while nargs and args[nargs-1] is Placeholder: nargs -= 1 args = args[:nargs] - np = args.count(Placeholder) + placeholder_count = args.count(Placeholder) if isinstance(func, partial): pargs = func.args - pnp = func.np + pnp = func.placeholder_count if pnp and args: all_args = list(pargs) nargs = len(args) @@ -312,39 +312,37 @@ def __new__(cls, func, /, *args, **keywords): pos += 1 if pnp < nargs: all_args.extend(args[pnp:]) - np += pnp - end + placeholder_count += pnp - end args = tuple(all_args) else: - np += pnp + placeholder_count += pnp args = func.args + args keywords = {**func.keywords, **keywords} func = func.func self = super(partial, cls).__new__(cls) - self.func = func self.args = args self.keywords = keywords - self.np = np + self.placeholder_count = placeholder_count return self def __call__(self, /, *args, **keywords): - if not hasattr(self, 'np'): - self.np = self.args.count(Placeholder) - if np := self.np: - if len(args) < np: - raise TypeError("unfilled placeholders in 'partial' call") + keywords = {**self.keywords, **keywords} + if placeholder_count := self.placeholder_count: + if len(args) < placeholder_count: + raise TypeError( + "missing positional arguments in 'partial' call; expected " + f"at least {placeholder_count}, got {len(fargs)}") f_args = list(self.args) j, pos = 0, 0 - while j < np: + while j < placeholder_count: pos = f_args.index(Placeholder, pos) f_args[pos] = args[j] j += 1 pos += 1 - keywords = {**self.keywords, **keywords} - return self.func(*f_args, *args[np:], **keywords) + return self.func(*f_args, *args[j:], **keywords) else: - keywords = {**self.keywords, **keywords} return self.func(*self.args, *args, **keywords) @recursive_repr() @@ -359,16 +357,18 @@ def __repr__(self): def __reduce__(self): return type(self), (self.func,), (self.func, self.args, - self.keywords or None, self.__dict__ or None) + self.keywords or None, self.placeholder_count, + self.__dict__ or None) def __setstate__(self, state): if not isinstance(state, tuple): raise TypeError("argument to __setstate__ must be a tuple") - if len(state) != 4: - raise TypeError(f"expected 4 items in state, got {len(state)}") - func, args, kwds, namespace = state + if len(state) != 5: + raise TypeError(f"expected 5 items in state, got {len(state)}") + func, args, kwds, placeholder_count, namespace = state if (not callable(func) or not isinstance(args, tuple) or (kwds is not None and not isinstance(kwds, dict)) or + not isinstance(placeholder_count, int) or (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") @@ -384,6 +384,7 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds + self.placeholder_count = placeholder_count try: from _functools import partial, Placeholder diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 055c123f61511f..b363f103cc59e4 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -57,13 +57,13 @@ static PyType_Slot placeholder_type_slots[] = { }; static PyType_Spec placeholder_type_spec = { - .name = "partial.Placeholder", + .name = "functools.Placeholder", .basicsize = sizeof(placeholderobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION, .slots = placeholder_type_slots -}; +}; // TODO> test: test.support.check_disallow_instantiation typedef struct { @@ -154,13 +154,14 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) Py_ssize_t nnp = 0; Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); PyObject *item; - if (nnargs > 0){ + if (nnargs > 0) { /* Trim placeholders from the end if needed */ Py_ssize_t nnargs_old = nnargs; for (; nnargs > 0; nnargs--) { item = PyTuple_GET_ITEM(nargs, nnargs-1); - if (!Py_Is(item, pto->placeholder)) + if (!Py_Is(item, pto->placeholder)) { break; + } } if (nnargs != nnargs_old) { PyObject *tmp = PyTuple_GetSlice(nargs, 0, nnargs); @@ -168,11 +169,12 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) nargs = tmp; } /* Count placeholders */ - if (nnargs > 1){ - for (Py_ssize_t i=0; i < nnargs - 1; i++){ + if (nnargs > 1) { + for (Py_ssize_t i=0; i < nnargs - 1; i++) { item = PyTuple_GET_ITEM(nargs, i); - if (Py_Is(item, pto->placeholder)) + if (Py_Is(item, pto->placeholder)) { nnp++; + } } } } @@ -185,12 +187,13 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) for (Py_ssize_t i=0, j=0; i < nfargs; i++) { if (i < npargs) { item = PyTuple_GET_ITEM(pargs, i); - if ((j < nnargs) & Py_Is(item, pto->placeholder)){ + if ((j < nnargs) & Py_Is(item, pto->placeholder)) { item = PyTuple_GET_ITEM(nargs, j); j++; pnp--; } - } else { + } + else { item = PyTuple_GET_ITEM(nargs, j); j++; } @@ -200,10 +203,12 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) pto->args = fargs; pto->np = pnp + nnp; Py_DECREF(nargs); - } else if (pargs == NULL) { + } + else if (pargs == NULL) { pto->args = nargs; pto->np = nnp; - } else { + } + else { pto->args = PySequence_Concat(pargs, nargs); pto->np = pnp + nnp; Py_DECREF(nargs); @@ -306,8 +311,9 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, } Py_ssize_t np = pto->np; if (nargs < np) { - PyErr_SetString(PyExc_TypeError, - "unfilled placeholders in 'partial' call"); + PyErr_Format(PyExc_TypeError, + "missing positional arguments in 'partial' call; " + "expected at least %zd, got %zd", np, nargs); return NULL; } @@ -358,19 +364,22 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, Py_ssize_t nargs_new; if (np) { nargs_new = pto_nargs + nargs - np; - Py_ssize_t j = 0; // Placeholder counter + Py_ssize_t j = 0; // New args index for (Py_ssize_t i=0; i < pto_nargs; i++) { if (Py_Is(pto_args[i], pto->placeholder)){ memcpy(stack + i, args + j, 1 * sizeof(PyObject*)); j += 1; - } else { + } + else { memcpy(stack + i, pto_args + i, 1 * sizeof(PyObject*)); } } - if (nargs_total > np){ - memcpy(stack + pto_nargs, args + np, (nargs_total - np) * sizeof(PyObject*)); + assert(j == np); + if (nargs_total > np) { + memcpy(stack + pto_nargs, args + j, (nargs_total - j) * sizeof(PyObject*)); } - } else { + } + else { nargs_new = pto_nargs + nargs; /* Copy to new stack, using borrowed references */ memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); @@ -411,8 +420,9 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t np = pto->np; if (nargs < np) { - PyErr_SetString(PyExc_TypeError, - "unfilled placeholders in 'partial' call"); + PyErr_Format(PyExc_TypeError, + "missing positional arguments in 'partial' call; " + "expected at least %zd, got %zd", np, nargs); return NULL; } @@ -445,25 +455,31 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); Py_ssize_t nargs_new = pto_nargs + nargs - np; args2 = PyTuple_New(nargs_new); + if (args2 == NULL) { + Py_XDECREF(kwargs2); + return NULL; + } PyObject *pto_args = pto->args; PyObject *item; - Py_ssize_t j = 0; // Placeholder counter + Py_ssize_t j = 0; // New args index for (Py_ssize_t i=0; i < pto_nargs; i++) { item = PyTuple_GET_ITEM(pto_args, i); - if (Py_Is(item, pto->placeholder)){ + if (Py_Is(item, pto->placeholder)) { item = PyTuple_GET_ITEM(args, j); j += 1; } PyTuple_SET_ITEM(args2, i, item); } - if (nargs > np){ + assert(j == np); + if (nargs > np) { for (Py_ssize_t i=pto_nargs; i < nargs_new; i++) { item = PyTuple_GET_ITEM(args, j); PyTuple_SET_ITEM(args2, i, item); j += 1; } } - } else { + } + else { /* Note: tupleconcat() is optimized for empty tuples */ args2 = PySequence_Concat(pto->args, args); } @@ -491,7 +507,7 @@ static PyMemberDef partial_memberlist[] = { "tuple of arguments to future partial calls"}, {"keywords", _Py_T_OBJECT, OFF(kw), Py_READONLY, "dictionary of keyword arguments to future partial calls"}, - {"np", Py_T_PYSSIZET, OFF(np), Py_READONLY, + {"placeholder_count", Py_T_PYSSIZET, OFF(np), Py_READONLY, "number of placeholders"}, {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(partialobject, weakreflist), Py_READONLY}, From 32bca19f046dae46a5a2a157fa991d45e58e4acf Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 8 Jun 2024 11:14:28 +0300 Subject: [PATCH 17/70] singleton sentinel and reduce --- Lib/functools.py | 2 +- Lib/test/test_functools.py | 40 +++++------ Modules/_functoolsmodule.c | 143 +++++++++++++++++++++++++++++-------- 3 files changed, 135 insertions(+), 50 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 7bcee5fdcf76cc..237bf15cd334dd 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -333,7 +333,7 @@ def __call__(self, /, *args, **keywords): if len(args) < placeholder_count: raise TypeError( "missing positional arguments in 'partial' call; expected " - f"at least {placeholder_count}, got {len(fargs)}") + f"at least {placeholder_count}, got {len(args)}") f_args = list(self.args) j, pos = 0, 0 while j < placeholder_count: diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 858b763ae4a258..7fa37c47479533 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -275,25 +275,25 @@ def test_recursive_repr(self): name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) - f.__setstate__((f, (), {}, {})) + f.__setstate__((f, (), {}, 0, {})) try: self.assertEqual(repr(f), '%s(...)' % (name,)) finally: - f.__setstate__((capture, (), {}, {})) + f.__setstate__((capture, (), {}, 0, {})) f = self.partial(capture) - f.__setstate__((capture, (f,), {}, {})) + f.__setstate__((capture, (f,), {}, 0, {})) try: self.assertEqual(repr(f), '%s(%r, ...)' % (name, capture,)) finally: - f.__setstate__((capture, (), {}, {})) + f.__setstate__((capture, (), {}, 0, {})) f = self.partial(capture) - f.__setstate__((capture, (), {'a': f}, {})) + f.__setstate__((capture, (), {'a': f}, 0, {})) try: self.assertEqual(repr(f), '%s(%r, a=...)' % (name, capture,)) finally: - f.__setstate__((capture, (), {}, {})) + f.__setstate__((capture, (), {}, 0, {})) def test_pickle(self): with replaced_module('functools', self.module): @@ -325,24 +325,24 @@ def test_deepcopy(self): def test_setstate(self): f = self.partial(signature) - f.__setstate__((capture, (1,), dict(a=10), dict(attr=[]))) + f.__setstate__((capture, (1,), dict(a=10), 0, dict(attr=[]))) self.assertEqual(signature(f), (capture, (1,), dict(a=10), dict(attr=[]))) self.assertEqual(f(2, b=20), ((1, 2), {'a': 10, 'b': 20})) - f.__setstate__((capture, (1,), dict(a=10), None)) + f.__setstate__((capture, (1,), dict(a=10), 0, None)) self.assertEqual(signature(f), (capture, (1,), dict(a=10), {})) self.assertEqual(f(2, b=20), ((1, 2), {'a': 10, 'b': 20})) - f.__setstate__((capture, (1,), None, None)) + f.__setstate__((capture, (1,), None, 0, None)) #self.assertEqual(signature(f), (capture, (1,), {}, {})) self.assertEqual(f(2, b=20), ((1, 2), {'b': 20})) self.assertEqual(f(2), ((1, 2), {})) self.assertEqual(f(), ((1,), {})) - f.__setstate__((capture, (), {}, None)) + f.__setstate__((capture, (), {}, 0, None)) self.assertEqual(signature(f), (capture, (), {}, {})) self.assertEqual(f(2, b=20), ((2,), {'b': 20})) self.assertEqual(f(2), ((2,), {})) @@ -360,7 +360,7 @@ def test_setstate_errors(self): def test_setstate_subclasses(self): f = self.partial(signature) - f.__setstate__((capture, MyTuple((1,)), MyDict(a=10), None)) + f.__setstate__((capture, MyTuple((1,)), MyDict(a=10), 0, None)) s = signature(f) self.assertEqual(s, (capture, (1,), dict(a=10), {})) self.assertIs(type(s[1]), tuple) @@ -370,7 +370,7 @@ def test_setstate_subclasses(self): self.assertIs(type(r[0]), tuple) self.assertIs(type(r[1]), dict) - f.__setstate__((capture, BadTuple((1,)), {}, None)) + f.__setstate__((capture, BadTuple((1,)), {}, 0, None)) s = signature(f) self.assertEqual(s, (capture, (1,), {}, {})) self.assertIs(type(s[1]), tuple) @@ -381,7 +381,7 @@ def test_setstate_subclasses(self): def test_recursive_pickle(self): with replaced_module('functools', self.module): f = self.partial(capture) - f.__setstate__((f, (), {}, {})) + f.__setstate__((f, (), {}, 0, {})) try: for proto in range(pickle.HIGHEST_PROTOCOL + 1): # gh-117008: Small limit since pickle uses C stack memory @@ -389,31 +389,31 @@ def test_recursive_pickle(self): with self.assertRaises(RecursionError): pickle.dumps(f, proto) finally: - f.__setstate__((capture, (), {}, {})) + f.__setstate__((capture, (), {}, 0, {})) f = self.partial(capture) - f.__setstate__((capture, (f,), {}, {})) + f.__setstate__((capture, (f,), {}, 0, {})) try: for proto in range(pickle.HIGHEST_PROTOCOL + 1): f_copy = pickle.loads(pickle.dumps(f, proto)) try: self.assertIs(f_copy.args[0], f_copy) finally: - f_copy.__setstate__((capture, (), {}, {})) + f_copy.__setstate__((capture, (), {}, 0, {})) finally: - f.__setstate__((capture, (), {}, {})) + f.__setstate__((capture, (), {}, 0, {})) f = self.partial(capture) - f.__setstate__((capture, (), {'a': f}, {})) + f.__setstate__((capture, (), {'a': f}, 0, {})) try: for proto in range(pickle.HIGHEST_PROTOCOL + 1): f_copy = pickle.loads(pickle.dumps(f, proto)) try: self.assertIs(f_copy.keywords['a'], f_copy) finally: - f_copy.__setstate__((capture, (), {}, {})) + f_copy.__setstate__((capture, (), {}, 0, {})) finally: - f.__setstate__((capture, (), {}, {})) + f.__setstate__((capture, (), {}, 0, {})) # Issue 6083: Reference counting bug def test_setstate_refcount(self): diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index b363f103cc59e4..ac130c368092e1 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -43,27 +43,118 @@ get_functools_state(PyObject *module) /* partial object **********************************************************/ -typedef struct { - PyObject_HEAD -} placeholderobject; +/* +Placeholder is an object that can be used to signal that positional + argument place is empty when using `partial` class +*/ +PyObject* +Py_GetPlaceholder(void); + +static PyObject * +placeholder_repr(PyObject *op) +{ + return PyUnicode_FromString("Placeholder"); +} -PyDoc_STRVAR(placeholder_doc, "placeholder for partial arguments"); +static void +placeholder_dealloc(PyObject* placeholder) +{ + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref None out of existence. Instead, + * since None is an immortal object, re-set the reference count. + */ + _Py_SetImmortal(placeholder); +} +static PyObject * +placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) { + PyErr_SetString(PyExc_TypeError, "PlaceholderType takes no arguments"); + return NULL; + } + return Py_GetPlaceholder(); +} -static PyType_Slot placeholder_type_slots[] = { - {Py_tp_doc, (void *)placeholder_doc}, - {0, 0} +static int +placeholder_bool(PyObject *v) +{ + PyErr_SetString(PyExc_TypeError, + "Placeholder should not be used in a boolean context"); + return -1; +} + +static PyNumberMethods placeholder_as_number = { + .nb_bool = placeholder_bool, }; -static PyType_Spec placeholder_type_spec = { - .name = "functools.Placeholder", - .basicsize = sizeof(placeholderobject), - .flags = Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_IMMUTABLETYPE | - Py_TPFLAGS_DISALLOW_INSTANTIATION, - .slots = placeholder_type_slots -}; // TODO> test: test.support.check_disallow_instantiation + +PyDoc_STRVAR(placeholder_doc, +"PlaceholderType()\n" +"--\n\n" +"The type of the Placeholder singleton."); + +PyTypeObject _PyPlaceholder_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "PlaceholderType", + 0, + 0, + placeholder_dealloc, /*tp_dealloc*/ /*never called*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + placeholder_repr, /*tp_repr*/ + &placeholder_as_number, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call */ + 0, /*tp_str */ + 0, /*tp_getattro */ + 0, /*tp_setattro */ + 0, /*tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /*tp_flags */ + placeholder_doc, /*tp_doc */ + 0, /*tp_traverse */ + 0, /*tp_clear */ + 0, /*tp_richcompare */ + 0, /*tp_weaklistoffset */ + 0, /*tp_iter */ + 0, /*tp_iternext */ + 0, /*tp_methods */ + 0, /*tp_members */ + 0, /*tp_getset */ + 0, /*tp_base */ + 0, /*tp_dict */ + 0, /*tp_descr_get */ + 0, /*tp_descr_set */ + 0, /*tp_dictoffset */ + 0, /*tp_init */ + 0, /*tp_alloc */ + placeholder_new, /*tp_new */ +}; + +PyObject _Py_PlaceholderStruct = _PyObject_HEAD_INIT(&_PyPlaceholder_Type); + +#define PY_CONSTANT_PLACEHOLDER 0 + +static PyObject* constants[] = { + &_Py_PlaceholderStruct, // PY_CONSTANT_PLACEHOLDER +}; + +PyObject* +Py_GetPlaceholder(void) +{ + if (PY_CONSTANT_PLACEHOLDER < Py_ARRAY_LENGTH(constants)) { + return constants[PY_CONSTANT_PLACEHOLDER]; + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} typedef struct { @@ -150,7 +241,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - pto->placeholder = (PyObject *) state->placeholder_type; + pto->placeholder = Py_GetPlaceholder(); Py_ssize_t nnp = 0; Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); PyObject *item; @@ -598,8 +689,8 @@ partial_repr(partialobject *pto) static PyObject * partial_reduce(partialobject *pto, PyObject *unused) { - return Py_BuildValue("O(O)(OOOO)", Py_TYPE(pto), pto->fn, pto->fn, - pto->args, pto->kw, + return Py_BuildValue("O(O)(OOOnO)", Py_TYPE(pto), pto->fn, pto->fn, + pto->args, pto->kw, pto->np, pto->dict ? pto->dict : Py_None); } @@ -607,9 +698,10 @@ static PyObject * partial_setstate(partialobject *pto, PyObject *state) { PyObject *fn, *fnargs, *kw, *dict; + Py_ssize_t np; if (!PyTuple_Check(state) || - !PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || + !PyArg_ParseTuple(state, "OOOnO", &fn, &fnargs, &kw, &np, &dict) || !PyCallable_Check(fn) || !PyTuple_Check(fnargs) || (kw != Py_None && !PyDict_Check(kw))) @@ -640,10 +732,10 @@ partial_setstate(partialobject *pto, PyObject *state) dict = NULL; else Py_INCREF(dict); - Py_SETREF(pto->fn, Py_NewRef(fn)); Py_SETREF(pto->args, fnargs); Py_SETREF(pto->kw, kw); + pto->np = np; Py_XSETREF(pto->dict, dict); partial_setvectorcall(pto); Py_RETURN_NONE; @@ -1626,6 +1718,7 @@ static PyType_Spec lru_cache_type_spec = { /* module level code ********************************************************/ + PyDoc_STRVAR(_functools_doc, "Tools that operate on functions."); @@ -1644,15 +1737,9 @@ _functools_exec(PyObject *module) return -1; } - state->placeholder_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, - &placeholder_type_spec, NULL); - if (state->placeholder_type == NULL) { - return -1; - } - if (PyModule_AddType(module, state->placeholder_type) < 0) { + if (PyModule_AddObject(module, "Placeholder", Py_GetPlaceholder()) < 0) { return -1; } - state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &partial_type_spec, NULL); if (state->partial_type == NULL) { @@ -1697,7 +1784,6 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) { _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); - Py_VISIT(state->placeholder_type); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1709,7 +1795,6 @@ _functools_clear(PyObject *module) { _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); - Py_CLEAR(state->placeholder_type); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); From 8576493083a57d059c27698e34a87b8138362d28 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 8 Jun 2024 11:31:01 +0300 Subject: [PATCH 18/70] python module sentinel better mimics the one of the extension --- Lib/functools.py | 23 +++++++++++++++++++++-- Modules/_functoolsmodule.c | 3 ++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 237bf15cd334dd..afd8392eff94ee 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -274,8 +274,27 @@ def reduce(function, sequence, initial=_initial_missing): ################################################################################ -class Placeholder: - """placeholder for partial arguments""" +class PlaceholderType: + """PlaceholderType() + -- + + The type of the Placeholder singleton. + Used as a placeholder for partial arguments. + """ + _instance = None + def __new__(cls): + if cls._instance is None: + cls._instance = object.__new__(cls) + return cls._instance + + def __repr__(self): + return 'Placeholder' + + def __bool__(self): + raise TypeError("Placeholder should not be used in a boolean context") + + +Placeholder = PlaceholderType() # Purely functional, no descriptor behaviour diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index ac130c368092e1..c68d282d4652e7 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -93,7 +93,8 @@ static PyNumberMethods placeholder_as_number = { PyDoc_STRVAR(placeholder_doc, "PlaceholderType()\n" "--\n\n" -"The type of the Placeholder singleton."); +"The type of the Placeholder singleton." +"Used as a placeholder for partial arguments."); PyTypeObject _PyPlaceholder_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) From a3fd2d66ef5b9a0ed811745431aacc27a1fe3624 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 8 Jun 2024 12:58:31 +0300 Subject: [PATCH 19/70] Emulated None behaviour, but using PyType_FromModuleAndSpec --- Modules/_functoolsmodule.c | 122 +++++++++++++++---------------------- 1 file changed, 49 insertions(+), 73 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index c68d282d4652e7..a2e881fc94620f 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -48,8 +48,21 @@ Placeholder is an object that can be used to signal that positional argument place is empty when using `partial` class */ -PyObject* -Py_GetPlaceholder(void); +#define PY_CONSTANT_PLACEHOLDER 0 + +static PyObject* constants[] = { + NULL, // PY_CONSTANT_PLACEHOLDER +}; + +typedef struct { + PyObject_HEAD +} placeholderobject; + +PyDoc_STRVAR(placeholder_doc, +"PlaceholderType()\n" +"--\n\n" +"The type of the Placeholder singleton.\n" +"Used as a placeholder for partial arguments."); static PyObject * placeholder_repr(PyObject *op) @@ -74,7 +87,12 @@ placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "PlaceholderType takes no arguments"); return NULL; } - return Py_GetPlaceholder(); + PyObject *singleton = constants[PY_CONSTANT_PLACEHOLDER]; + if (singleton == NULL) { + singleton = PyType_GenericNew(type, NULL, NULL); + constants[PY_CONSTANT_PLACEHOLDER] = singleton; + } + return singleton; } static int @@ -85,78 +103,23 @@ placeholder_bool(PyObject *v) return -1; } -static PyNumberMethods placeholder_as_number = { - .nb_bool = placeholder_bool, -}; - - -PyDoc_STRVAR(placeholder_doc, -"PlaceholderType()\n" -"--\n\n" -"The type of the Placeholder singleton." -"Used as a placeholder for partial arguments."); - -PyTypeObject _PyPlaceholder_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "PlaceholderType", - 0, - 0, - placeholder_dealloc, /*tp_dealloc*/ /*never called*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - placeholder_repr, /*tp_repr*/ - &placeholder_as_number, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call */ - 0, /*tp_str */ - 0, /*tp_getattro */ - 0, /*tp_setattro */ - 0, /*tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /*tp_flags */ - placeholder_doc, /*tp_doc */ - 0, /*tp_traverse */ - 0, /*tp_clear */ - 0, /*tp_richcompare */ - 0, /*tp_weaklistoffset */ - 0, /*tp_iter */ - 0, /*tp_iternext */ - 0, /*tp_methods */ - 0, /*tp_members */ - 0, /*tp_getset */ - 0, /*tp_base */ - 0, /*tp_dict */ - 0, /*tp_descr_get */ - 0, /*tp_descr_set */ - 0, /*tp_dictoffset */ - 0, /*tp_init */ - 0, /*tp_alloc */ - placeholder_new, /*tp_new */ +static PyType_Slot placeholder_type_slots[] = { + {Py_tp_dealloc, placeholder_dealloc}, + {Py_tp_repr, placeholder_repr}, + {Py_tp_doc, (void *)placeholder_doc}, + {Py_tp_new, placeholder_new}, + // Number protocol + {Py_nb_bool, placeholder_bool}, + {0, 0} }; -PyObject _Py_PlaceholderStruct = _PyObject_HEAD_INIT(&_PyPlaceholder_Type); - -#define PY_CONSTANT_PLACEHOLDER 0 - -static PyObject* constants[] = { - &_Py_PlaceholderStruct, // PY_CONSTANT_PLACEHOLDER +static PyType_Spec placeholder_type_spec = { + .name = "partial.PlaceholderType", + .basicsize = sizeof(placeholderobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, + .slots = placeholder_type_slots }; -PyObject* -Py_GetPlaceholder(void) -{ - if (PY_CONSTANT_PLACEHOLDER < Py_ARRAY_LENGTH(constants)) { - return constants[PY_CONSTANT_PLACEHOLDER]; - } - else { - PyErr_BadInternalCall(); - return NULL; - } -} - typedef struct { PyObject_HEAD @@ -242,7 +205,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - pto->placeholder = Py_GetPlaceholder(); + pto->placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); Py_ssize_t nnp = 0; Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); PyObject *item; @@ -1738,7 +1701,18 @@ _functools_exec(PyObject *module) return -1; } - if (PyModule_AddObject(module, "Placeholder", Py_GetPlaceholder()) < 0) { + // if (PyModule_AddObject(module, "Placeholder", Py_GetPlaceholder()) < 0) { + // return -1; + // } + state->placeholder_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &placeholder_type_spec, NULL); + if (state->placeholder_type == NULL) { + return -1; + } + if (PyModule_AddType(module, state->placeholder_type) < 0) { + return -1; + } + if (PyModule_AddObject(module, "Placeholder", PyObject_CallNoArgs((PyObject *)state->placeholder_type)) < 0) { return -1; } state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, @@ -1785,6 +1759,7 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg) { _functools_state *state = get_functools_state(module); Py_VISIT(state->kwd_mark); + Py_VISIT(state->placeholder_type); Py_VISIT(state->partial_type); Py_VISIT(state->keyobject_type); Py_VISIT(state->lru_list_elem_type); @@ -1796,6 +1771,7 @@ _functools_clear(PyObject *module) { _functools_state *state = get_functools_state(module); Py_CLEAR(state->kwd_mark); + Py_CLEAR(state->placeholder_type); Py_CLEAR(state->partial_type); Py_CLEAR(state->keyobject_type); Py_CLEAR(state->lru_list_elem_type); From 0852993701900ac6afe706d60416ad98e4a88d53 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 8 Jun 2024 16:15:32 +0300 Subject: [PATCH 20/70] review feedback --- Doc/library/functools.rst | 6 +++--- Lib/functools.py | 14 +++++++++----- Lib/test/test_functools.py | 7 +++++++ Modules/_functoolsmodule.c | 9 +++------ 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 47f3d13f560489..3ec916621aa460 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -353,8 +353,8 @@ The :mod:`functools` module defines the following functions: f"got {len(fargs)}") newargs = list(args) j = 0 - for i in range(len(args)): - if args[i] is Placeholder: + for i, arg in enumarate(args): + if arg is Placeholder: newargs[i] = fargs[j] j += 1 newargs.extend(fargs[j:]) @@ -381,7 +381,7 @@ The :mod:`functools` module defines the following functions: If ``Placeholder`` sentinels are present in *args*, they will be filled first when :func:`partial` is called. This allows custom selection of positional arguments to be pre-filled when constructing :ref:`partial object`. - If ``Placeholder`` sentinels are used, all of them must be filled at call time.: + If ``Placeholder`` sentinels are used, all of them must be filled at call time: >>> from functools import partial, Placeholder >>> say_to_world = partial(print, Placeholder, 'world!') diff --git a/Lib/functools.py b/Lib/functools.py index afd8392eff94ee..c210ac016a55de 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -281,11 +281,12 @@ class PlaceholderType: The type of the Placeholder singleton. Used as a placeholder for partial arguments. """ - _instance = None + _singleton = None + def __new__(cls): - if cls._instance is None: - cls._instance = object.__new__(cls) - return cls._instance + if cls._singleton is None: + cls._singleton = object.__new__(cls) + return cls._singleton def __repr__(self): return 'Placeholder' @@ -293,6 +294,9 @@ def __repr__(self): def __bool__(self): raise TypeError("Placeholder should not be used in a boolean context") + def __reduce__(self): + return type(self), () + Placeholder = PlaceholderType() @@ -406,7 +410,7 @@ def __setstate__(self, state): self.placeholder_count = placeholder_count try: - from _functools import partial, Placeholder + from _functools import partial, PlaceholderType, Placeholder except ImportError: pass diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 7fa37c47479533..226a2c5cb39724 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -247,6 +247,13 @@ def test_placeholders_optimization(self): expected = (-1, 0, 1, 2, 3, 4, 5) self.assertTrue(expected == got and empty == {}) + def test_construct_placeholder_singleton(self): + PH = self.module.Placeholder + tp = type(PH) + self.assertIs(tp(), PH) + self.assertRaises(TypeError, tp, 1, 2) + self.assertRaises(TypeError, tp, a=1, b=2) + def test_repr(self): args = (object(), object()) args_repr = ', '.join(repr(a) for a in args) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index a2e881fc94620f..7f3cc1d274ce4b 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -74,8 +74,8 @@ static void placeholder_dealloc(PyObject* placeholder) { /* This should never get called, but we also don't want to SEGV if - * we accidentally decref None out of existence. Instead, - * since None is an immortal object, re-set the reference count. + * we accidentally decref Placeholder out of existence. Instead, + * since Placeholder is an immortal object, re-set the reference count. */ _Py_SetImmortal(placeholder); } @@ -114,7 +114,7 @@ static PyType_Slot placeholder_type_slots[] = { }; static PyType_Spec placeholder_type_spec = { - .name = "partial.PlaceholderType", + .name = "functools.PlaceholderType", .basicsize = sizeof(placeholderobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, .slots = placeholder_type_slots @@ -1701,9 +1701,6 @@ _functools_exec(PyObject *module) return -1; } - // if (PyModule_AddObject(module, "Placeholder", Py_GetPlaceholder()) < 0) { - // return -1; - // } state->placeholder_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &placeholder_type_spec, NULL); if (state->placeholder_type == NULL) { From 6fea3480019a04d5dedf345bd294e2e10e39b502 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 8 Jun 2024 17:07:16 +0300 Subject: [PATCH 21/70] included constant into tsv --- Tools/c-analyzer/cpython/ignored.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index a3bdf0396fd3e1..ebf7cc8fcf3590 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -227,6 +227,7 @@ Modules/_decimal/_decimal.c - signal_map_template - Modules/_decimal/_decimal.c - ssize_constants - Modules/_decimal/_decimal.c - INVALID_SIGNALDICT_ERROR_MSG - Modules/_elementtree.c - ExpatMemoryHandler - +Modules/_functoolsmodule.c - constants - Modules/_hashopenssl.c - py_hashes - Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - Modules/_hacl/Hacl_Hash_MD5.c - _h0 - From caec6e8cdd2a93d730e597ba280c035f7f5db603 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sat, 8 Jun 2024 17:23:31 +0300 Subject: [PATCH 22/70] documentation update --- Doc/library/functools.rst | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3ec916621aa460..3cdc117125763a 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -378,20 +378,31 @@ The :mod:`functools` module defines the following functions: >>> basetwo('10010') 18 - If ``Placeholder`` sentinels are present in *args*, they will be filled first + If :data:`Placeholder` sentinels are present in *args*, they will be filled first when :func:`partial` is called. This allows custom selection of positional arguments to be pre-filled when constructing :ref:`partial object`. - If ``Placeholder`` sentinels are used, all of them must be filled at call time: + If :data:`Placeholder` sentinels are used, all of them must be filled at call time: >>> from functools import partial, Placeholder - >>> say_to_world = partial(print, Placeholder, 'world!') + >>> say_to_world = partial(print, Placeholder, Placeholder, "world!") >>> say_to_world.placeholder_count - 1 - >>> say_to_world('Hello') - Hello world! + 2 + >>> say_to_world('Hello', 'dear') + Hello dear world! + + Calling ``say_to_world('Hello')`` would result in :exc:`TypeError`. .. versionchanged:: 3.14 - Support for ``Placeholder`` in *args* + Support for :data:`Placeholder` in *args* + +.. data:: Placeholder + + This is a singleton object that is used as a sentinel for representing a + gap in positional arguments when calling :func:`partial`. + It has similar features as :data:`None`: + + >>> type(Placeholder)() is Placeholder + True .. class:: partialmethod(func, /, *args, **keywords) From 115b8c5cb335c9cb69b9d394ae0e622913cfbc6a Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sun, 9 Jun 2024 20:36:38 +0300 Subject: [PATCH 23/70] review edits --- Doc/library/functools.rst | 13 +++--- Lib/functools.py | 46 ++++++++++--------- ...-05-25-00-54-26.gh-issue-119127.LpPvag.rst | 4 +- Modules/_functoolsmodule.c | 19 +++++--- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3cdc117125763a..c7045ab644379e 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -330,7 +330,7 @@ The :mod:`functools` module defines the following functions: .. function:: partial(func, /, *args, **keywords) - Return a new :ref:`partial object` which when called + Return a new :ref:`partial object ` which when called will behave like *func* called with the positional arguments *args* and keyword arguments *keywords*. If more arguments are supplied to the call, they are appended to *args*. If additional keyword arguments are @@ -380,7 +380,8 @@ The :mod:`functools` module defines the following functions: If :data:`Placeholder` sentinels are present in *args*, they will be filled first when :func:`partial` is called. This allows custom selection of positional arguments - to be pre-filled when constructing :ref:`partial object`. + to be pre-filled when constructing a :ref:`partial object `. + If :data:`Placeholder` sentinels are used, all of them must be filled at call time: >>> from functools import partial, Placeholder @@ -390,16 +391,16 @@ The :mod:`functools` module defines the following functions: >>> say_to_world('Hello', 'dear') Hello dear world! - Calling ``say_to_world('Hello')`` would result in :exc:`TypeError`. + Calling ``say_to_world('Hello')`` would raise a :exc:`TypeError`. .. versionchanged:: 3.14 Support for :data:`Placeholder` in *args* .. data:: Placeholder - This is a singleton object that is used as a sentinel for representing a - gap in positional arguments when calling :func:`partial`. - It has similar features as :data:`None`: + A singleton object used as a sentinel to represent a + "gap" in positional arguments when calling :func:`partial`. + This object has features similar to ``None``: >>> type(Placeholder)() is Placeholder True diff --git a/Lib/functools.py b/Lib/functools.py index c210ac016a55de..80f058ccb3aaec 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -281,12 +281,12 @@ class PlaceholderType: The type of the Placeholder singleton. Used as a placeholder for partial arguments. """ - _singleton = None + __instance = None def __new__(cls): - if cls._singleton is None: - cls._singleton = object.__new__(cls) - return cls._singleton + if cls.__instance is None: + cls.__instance = object.__new__(cls) + return cls.__instance def __repr__(self): return 'Placeholder' @@ -313,13 +313,13 @@ class partial: def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") - placeholder_count = 0 + np = 0 nargs = len(args) if args: - while nargs and args[nargs-1] is Placeholder: + while nargs and args[nargs - 1] is Placeholder: nargs -= 1 args = args[:nargs] - placeholder_count = args.count(Placeholder) + np = args.count(Placeholder) if isinstance(func, partial): pargs = func.args pnp = func.placeholder_count @@ -335,10 +335,10 @@ def __new__(cls, func, /, *args, **keywords): pos += 1 if pnp < nargs: all_args.extend(args[pnp:]) - placeholder_count += pnp - end + np += pnp - end args = tuple(all_args) else: - placeholder_count += pnp + np += pnp args = func.args + args keywords = {**func.keywords, **keywords} func = func.func @@ -347,26 +347,28 @@ def __new__(cls, func, /, *args, **keywords): self.func = func self.args = args self.keywords = keywords - self.placeholder_count = placeholder_count + self.placeholder_count = np return self def __call__(self, /, *args, **keywords): - keywords = {**self.keywords, **keywords} - if placeholder_count := self.placeholder_count: - if len(args) < placeholder_count: + np = self.placeholder_count + p_args = self.args + if np: + if len(args) < np: raise TypeError( - "missing positional arguments in 'partial' call; expected " - f"at least {placeholder_count}, got {len(args)}") - f_args = list(self.args) + "missing positional arguments " + "in 'partial' call; expected " + f"at least {np}, got {len(args)}") + p_args = list(p_args) j, pos = 0, 0 - while j < placeholder_count: - pos = f_args.index(Placeholder, pos) - f_args[pos] = args[j] + while j < np: + pos = p_args.index(Placeholder, pos) + p_args[pos] = args[j] j += 1 pos += 1 - return self.func(*f_args, *args[j:], **keywords) - else: - return self.func(*self.args, *args, **keywords) + args = args[j:] + keywords = {**self.keywords, **keywords} + return self.func(*p_args, *args, **keywords) @recursive_repr() def __repr__(self): diff --git a/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst b/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst index 70bb0ce98bb555..e47e2ae89dbff0 100644 --- a/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst +++ b/Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst @@ -1,2 +1,2 @@ -``partial`` of ``functools`` module now supports placeholders for positional -arguments. +Positional arguments of :func:`functools.partial` objects +now support placeholders via :data:`functools.Placeholder`. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 7f3cc1d274ce4b..4d9d2a59c9fb32 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -26,6 +26,7 @@ typedef struct _functools_state { /* this object is used delimit args and keywords in the cache keys */ PyObject *kwd_mark; PyTypeObject *placeholder_type; + PyObject *placeholder; PyTypeObject *partial_type; PyTypeObject *keyobject_type; PyTypeObject *lru_list_elem_type; @@ -205,7 +206,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - pto->placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); + pto->placeholder = state->placeholder; Py_ssize_t nnp = 0; Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); PyObject *item; @@ -517,7 +518,7 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) PyObject *pto_args = pto->args; PyObject *item; Py_ssize_t j = 0; // New args index - for (Py_ssize_t i=0; i < pto_nargs; i++) { + for (Py_ssize_t i = 0; i < pto_nargs; i++) { item = PyTuple_GET_ITEM(pto_args, i); if (Py_Is(item, pto->placeholder)) { item = PyTuple_GET_ITEM(args, j); @@ -537,10 +538,10 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) else { /* Note: tupleconcat() is optimized for empty tuples */ args2 = PySequence_Concat(pto->args, args); - } - if (args2 == NULL) { - Py_XDECREF(kwargs2); - return NULL; + if (args2 == NULL) { + Py_XDECREF(kwargs2); + return NULL; + } } PyObject *res = PyObject_Call(pto->fn, args2, kwargs2); @@ -1709,7 +1710,11 @@ _functools_exec(PyObject *module) if (PyModule_AddType(module, state->placeholder_type) < 0) { return -1; } - if (PyModule_AddObject(module, "Placeholder", PyObject_CallNoArgs((PyObject *)state->placeholder_type)) < 0) { + state->placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type); + if (state->placeholder == NULL) { + return -1; + } + if (PyModule_AddObject(module, "Placeholder", state->placeholder) < 0) { return -1; } state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, From 3f5f00bbd3f9f7a4c33a09a779bbfe4c44ce04d9 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 10 Jun 2024 02:23:42 +0300 Subject: [PATCH 24/70] trailing placeholder prohibition and small changes --- Doc/library/functools.rst | 2 +- Lib/functools.py | 25 +++++------ Lib/test/test_functools.py | 12 ++---- Modules/_functoolsmodule.c | 87 +++++++++++++++----------------------- 4 files changed, 50 insertions(+), 76 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index c7045ab644379e..10f43edcfa3e5f 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -382,7 +382,7 @@ The :mod:`functools` module defines the following functions: when :func:`partial` is called. This allows custom selection of positional arguments to be pre-filled when constructing a :ref:`partial object `. - If :data:`Placeholder` sentinels are used, all of them must be filled at call time: + If :data:`!Placeholder` sentinels are used, all of them must be filled at call time: >>> from functools import partial, Placeholder >>> say_to_world = partial(print, Placeholder, Placeholder, "world!") diff --git a/Lib/functools.py b/Lib/functools.py index 80f058ccb3aaec..9e7fe913fcc0ad 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -275,10 +275,8 @@ def reduce(function, sequence, initial=_initial_missing): class PlaceholderType: - """PlaceholderType() - -- + """The type of the Placeholder singleton. - The type of the Placeholder singleton. Used as a placeholder for partial arguments. """ __instance = None @@ -288,14 +286,14 @@ def __new__(cls): cls.__instance = object.__new__(cls) return cls.__instance - def __repr__(self): - return 'Placeholder' - def __bool__(self): raise TypeError("Placeholder should not be used in a boolean context") + def __repr__(self): + return 'Placeholder' + def __reduce__(self): - return type(self), () + return 'Placeholder' Placeholder = PlaceholderType() @@ -314,25 +312,24 @@ def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") np = 0 - nargs = len(args) if args: - while nargs and args[nargs - 1] is Placeholder: - nargs -= 1 - args = args[:nargs] - np = args.count(Placeholder) + if args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + np = args[:-1].count(Placeholder) if isinstance(func, partial): pargs = func.args pnp = func.placeholder_count + # merge args with args of `func` which is `partial` if pnp and args: all_args = list(pargs) nargs = len(args) - j, pos = 0, 0 + pos, j = 0, 0 end = nargs if nargs < pnp else pnp while j < end: pos = all_args.index(Placeholder, pos) all_args[pos] = args[j] - j += 1 pos += 1 + j += 1 if pnp < nargs: all_args.extend(args[pnp:]) np += pnp - end diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 226a2c5cb39724..5aeb9ba377b59a 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -210,15 +210,11 @@ def foo(bar): p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') - def test_placeholders_trailing_trim(self): + def test_placeholders_trailing_raise(self): PH = self.module.Placeholder - for args in [(PH,), (PH, 0), (0, PH), (0, PH, 1, PH, PH, PH)]: - expected, n = tuple(args), len(args) - while n and args[n-1] is PH: - n -= 1 - expected = expected[:n] - p = self.partial(capture, *args) - self.assertTrue(p.args == expected) + for args in [(PH,), (0, PH), (0, PH, 1, PH, PH, PH)]: + with self.assertRaises(TypeError): + self.partial(capture, *args) def test_placeholders(self): PH = self.module.Placeholder diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 4d9d2a59c9fb32..1368648b5625f9 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -44,25 +44,17 @@ get_functools_state(PyObject *module) /* partial object **********************************************************/ -/* -Placeholder is an object that can be used to signal that positional - argument place is empty when using `partial` class -*/ - -#define PY_CONSTANT_PLACEHOLDER 0 +// The 'Placeholder' singleton indicates which formal positional +// parameters are to be bound first when using a 'partial' object. -static PyObject* constants[] = { - NULL, // PY_CONSTANT_PLACEHOLDER -}; +static PyObject* placeholder_instance; typedef struct { PyObject_HEAD } placeholderobject; PyDoc_STRVAR(placeholder_doc, -"PlaceholderType()\n" -"--\n\n" -"The type of the Placeholder singleton.\n" +"The type of the Placeholder singleton.\n\n" "Used as a placeholder for partial arguments."); static PyObject * @@ -88,12 +80,10 @@ placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "PlaceholderType takes no arguments"); return NULL; } - PyObject *singleton = constants[PY_CONSTANT_PLACEHOLDER]; - if (singleton == NULL) { - singleton = PyType_GenericNew(type, NULL, NULL); - constants[PY_CONSTANT_PLACEHOLDER] = singleton; + if (placeholder_instance == NULL) { + placeholder_instance = PyType_GenericNew(type, NULL, NULL); } - return singleton; + return placeholder_instance; } static int @@ -156,12 +146,20 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) PyObject *func, *pargs, *nargs, *pkw; partialobject *pto; Py_ssize_t pnp = 0; + Py_ssize_t nnargs = PyTuple_GET_SIZE(args); - if (PyTuple_GET_SIZE(args) < 1) { + if (nnargs < 1) { PyErr_SetString(PyExc_TypeError, "type 'partial' takes at least one argument"); return NULL; } + nnargs--; + func = PyTuple_GET_ITEM(args, 0); + if (!PyCallable_Check(func)) { + PyErr_SetString(PyExc_TypeError, + "the first argument must be callable"); + return NULL; + } _functools_state *state = get_functools_state_by_type(type); if (state == NULL) { @@ -169,8 +167,6 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } pargs = pkw = NULL; - func = PyTuple_GET_ITEM(args, 0); - int res = PyObject_TypeCheck(func, state->partial_type); if (res == -1) { return NULL; @@ -187,11 +183,6 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) assert(PyDict_Check(pkw)); } } - if (!PyCallable_Check(func)) { - PyErr_SetString(PyExc_TypeError, - "the first argument must be callable"); - return NULL; - } /* create partialobject structure */ pto = (partialobject *)type->tp_alloc(type, 0); @@ -200,47 +191,38 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) pto->fn = Py_NewRef(func); - nargs = PyTuple_GetSlice(args, 1, PY_SSIZE_T_MAX); + pto->placeholder = state->placeholder; + if (Py_Is(PyTuple_GET_ITEM(args, nnargs), pto->placeholder)) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + + nargs = PyTuple_GetSlice(args, 1, nnargs + 1); if (nargs == NULL) { Py_DECREF(pto); return NULL; } - pto->placeholder = state->placeholder; Py_ssize_t nnp = 0; - Py_ssize_t nnargs = PyTuple_GET_SIZE(nargs); PyObject *item; - if (nnargs > 0) { - /* Trim placeholders from the end if needed */ - Py_ssize_t nnargs_old = nnargs; - for (; nnargs > 0; nnargs--) { - item = PyTuple_GET_ITEM(nargs, nnargs-1); - if (!Py_Is(item, pto->placeholder)) { - break; - } - } - if (nnargs != nnargs_old) { - PyObject *tmp = PyTuple_GetSlice(nargs, 0, nnargs); - Py_DECREF(nargs); - nargs = tmp; - } - /* Count placeholders */ - if (nnargs > 1) { - for (Py_ssize_t i=0; i < nnargs - 1; i++) { - item = PyTuple_GET_ITEM(nargs, i); - if (Py_Is(item, pto->placeholder)) { - nnp++; - } + /* Count placeholders */ + if (nnargs > 1) { + for (Py_ssize_t i = 0; i < nnargs - 1; i++) { + item = PyTuple_GET_ITEM(nargs, i); + if (Py_Is(item, pto->placeholder)) { + nnp++; } } } + /* merge args with args of `func` which is `partial` */ if ((pnp > 0) && (nnargs > 0)) { Py_ssize_t npargs = PyTuple_GET_SIZE(pargs); Py_ssize_t nfargs = npargs; if (nnargs > pnp) nfargs += nnargs - pnp; PyObject *fargs = PyTuple_New(nfargs); - for (Py_ssize_t i=0, j=0; i < nfargs; i++) { + for (Py_ssize_t i = 0, j = 0; i < nfargs; i++) { if (i < npargs) { item = PyTuple_GET_ITEM(pargs, i); if ((j < nnargs) & Py_Is(item, pto->placeholder)) { @@ -421,7 +403,7 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, if (np) { nargs_new = pto_nargs + nargs - np; Py_ssize_t j = 0; // New args index - for (Py_ssize_t i=0; i < pto_nargs; i++) { + for (Py_ssize_t i = 0; i < pto_nargs; i++) { if (Py_Is(pto_args[i], pto->placeholder)){ memcpy(stack + i, args + j, 1 * sizeof(PyObject*)); j += 1; @@ -528,7 +510,7 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) } assert(j == np); if (nargs > np) { - for (Py_ssize_t i=pto_nargs; i < nargs_new; i++) { + for (Py_ssize_t i = pto_nargs; i < nargs_new; i++) { item = PyTuple_GET_ITEM(args, j); PyTuple_SET_ITEM(args2, i, item); j += 1; @@ -1683,7 +1665,6 @@ static PyType_Spec lru_cache_type_spec = { /* module level code ********************************************************/ - PyDoc_STRVAR(_functools_doc, "Tools that operate on functions."); From 202c9295f75d564057c4d5d9c40380f38f78faba Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 10 Jun 2024 02:50:25 +0300 Subject: [PATCH 25/70] change constant name in ignores --- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index ebf7cc8fcf3590..d03a5436996075 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -227,7 +227,7 @@ Modules/_decimal/_decimal.c - signal_map_template - Modules/_decimal/_decimal.c - ssize_constants - Modules/_decimal/_decimal.c - INVALID_SIGNALDICT_ERROR_MSG - Modules/_elementtree.c - ExpatMemoryHandler - -Modules/_functoolsmodule.c - constants - +Modules/_functoolsmodule.c - placeholder_instance - Modules/_hashopenssl.c - py_hashes - Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - Modules/_hacl/Hacl_Hash_MD5.c - _h0 - From 2c16d38600ea8edebf04368d4f1ee6614007efae Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 11 Jun 2024 04:52:00 +0300 Subject: [PATCH 26/70] PlaceholderType Hidden --- Lib/functools.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 9e7fe913fcc0ad..24fd5ef992836c 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -274,7 +274,7 @@ def reduce(function, sequence, initial=_initial_missing): ################################################################################ -class PlaceholderType: +class __PlaceholderTypeBase: """The type of the Placeholder singleton. Used as a placeholder for partial arguments. @@ -296,7 +296,7 @@ def __reduce__(self): return 'Placeholder' -Placeholder = PlaceholderType() +Placeholder = type('PlaceholderType', (__PlaceholderTypeBase,), {})() # Purely functional, no descriptor behaviour @@ -311,11 +311,12 @@ class partial: def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") - np = 0 if args: if args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") - np = args[:-1].count(Placeholder) + np = args.count(Placeholder) + else: + np = 0 if isinstance(func, partial): pargs = func.args pnp = func.placeholder_count @@ -323,7 +324,7 @@ def __new__(cls, func, /, *args, **keywords): if pnp and args: all_args = list(pargs) nargs = len(args) - pos, j = 0, 0 + pos = j = 0 end = nargs if nargs < pnp else pnp while j < end: pos = all_args.index(Placeholder, pos) @@ -351,19 +352,20 @@ def __call__(self, /, *args, **keywords): np = self.placeholder_count p_args = self.args if np: - if len(args) < np: + n = len(args) + if n < np: raise TypeError( "missing positional arguments " "in 'partial' call; expected " - f"at least {np}, got {len(args)}") + f"at least {np}, got {n}") p_args = list(p_args) - j, pos = 0, 0 + pos = j = 0 while j < np: pos = p_args.index(Placeholder, pos) p_args[pos] = args[j] - j += 1 pos += 1 - args = args[j:] + j += 1 + args = args[np:] if n > np else () keywords = {**self.keywords, **keywords} return self.func(*p_args, *args, **keywords) @@ -409,7 +411,7 @@ def __setstate__(self, state): self.placeholder_count = placeholder_count try: - from _functools import partial, PlaceholderType, Placeholder + from _functools import partial, Placeholder except ImportError: pass From 400ff55b75e981a8eaeeeab18e92e10cbbdcf807 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 20 Jun 2024 12:07:29 +0300 Subject: [PATCH 27/70] support 4-arg pre-placeholder state --- Lib/functools.py | 11 ++++++-- Lib/test/test_functools.py | 53 ++++++++++++++++++++++++-------------- Modules/_functoolsmodule.c | 22 +++++++++++++--- 3 files changed, 61 insertions(+), 25 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 24fd5ef992836c..7bd962f7a7bab5 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -387,9 +387,16 @@ def __reduce__(self): def __setstate__(self, state): if not isinstance(state, tuple): raise TypeError("argument to __setstate__ must be a tuple") - if len(state) != 5: + n = len(state) + if n == 4: + # Support pre-placeholder de-serialization + func, args, kwds, namespace = state + placeholder_count = 0 + elif n == 5: + func, args, kwds, placeholder_count, namespace = state + else: raise TypeError(f"expected 5 items in state, got {len(state)}") - func, args, kwds, placeholder_count, namespace = state + if (not callable(func) or not isinstance(args, tuple) or (kwds is not None and not isinstance(kwds, dict)) or not isinstance(placeholder_count, int) or diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 5aeb9ba377b59a..c401f30b996015 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -278,25 +278,25 @@ def test_recursive_repr(self): name = f"{self.partial.__module__}.{self.partial.__qualname__}" f = self.partial(capture) - f.__setstate__((f, (), {}, 0, {})) + f.__setstate__((f, (), {}, {})) try: self.assertEqual(repr(f), '%s(...)' % (name,)) finally: - f.__setstate__((capture, (), {}, 0, {})) + f.__setstate__((capture, (), {}, {})) f = self.partial(capture) - f.__setstate__((capture, (f,), {}, 0, {})) + f.__setstate__((capture, (f,), {}, {})) try: self.assertEqual(repr(f), '%s(%r, ...)' % (name, capture,)) finally: - f.__setstate__((capture, (), {}, 0, {})) + f.__setstate__((capture, (), {}, {})) f = self.partial(capture) - f.__setstate__((capture, (), {'a': f}, 0, {})) + f.__setstate__((capture, (), {'a': f}, {})) try: self.assertEqual(repr(f), '%s(%r, a=...)' % (name, capture,)) finally: - f.__setstate__((capture, (), {}, 0, {})) + f.__setstate__((capture, (), {}, {})) def test_pickle(self): with replaced_module('functools', self.module): @@ -328,29 +328,42 @@ def test_deepcopy(self): def test_setstate(self): f = self.partial(signature) - f.__setstate__((capture, (1,), dict(a=10), 0, dict(attr=[]))) + f.__setstate__((capture, (1,), dict(a=10), dict(attr=[]))) self.assertEqual(signature(f), (capture, (1,), dict(a=10), dict(attr=[]))) self.assertEqual(f(2, b=20), ((1, 2), {'a': 10, 'b': 20})) - f.__setstate__((capture, (1,), dict(a=10), 0, None)) + f.__setstate__((capture, (1,), dict(a=10), None)) self.assertEqual(signature(f), (capture, (1,), dict(a=10), {})) self.assertEqual(f(2, b=20), ((1, 2), {'a': 10, 'b': 20})) - f.__setstate__((capture, (1,), None, 0, None)) + f.__setstate__((capture, (1,), None, None)) #self.assertEqual(signature(f), (capture, (1,), {}, {})) self.assertEqual(f(2, b=20), ((1, 2), {'b': 20})) self.assertEqual(f(2), ((1, 2), {})) self.assertEqual(f(), ((1,), {})) - f.__setstate__((capture, (), {}, 0, None)) + f.__setstate__((capture, (), {}, None)) self.assertEqual(signature(f), (capture, (), {}, {})) self.assertEqual(f(2, b=20), ((2,), {'b': 20})) self.assertEqual(f(2), ((2,), {})) self.assertEqual(f(), ((), {})) + # Set State with placeholders + f = self.partial(signature) + f.__setstate__((capture, (1,), dict(a=10), 0, dict(attr=[]))) + self.assertEqual(signature(f), (capture, (1,), dict(a=10), dict(attr=[]))) + + PH = self.module.Placeholder + f = self.partial(signature) + f.__setstate__((capture, (PH, 1), dict(a=10), 1, dict(attr=[]))) + self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[]))) + with self.assertRaises(TypeError): + self.assertEqual(f(), (PH, 1), dict(a=10)) + self.assertEqual(f(2), ((2, 1), dict(a=10))) + def test_setstate_errors(self): f = self.partial(signature) self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) @@ -363,7 +376,7 @@ def test_setstate_errors(self): def test_setstate_subclasses(self): f = self.partial(signature) - f.__setstate__((capture, MyTuple((1,)), MyDict(a=10), 0, None)) + f.__setstate__((capture, MyTuple((1,)), MyDict(a=10), None)) s = signature(f) self.assertEqual(s, (capture, (1,), dict(a=10), {})) self.assertIs(type(s[1]), tuple) @@ -373,7 +386,7 @@ def test_setstate_subclasses(self): self.assertIs(type(r[0]), tuple) self.assertIs(type(r[1]), dict) - f.__setstate__((capture, BadTuple((1,)), {}, 0, None)) + f.__setstate__((capture, BadTuple((1,)), {}, None)) s = signature(f) self.assertEqual(s, (capture, (1,), {}, {})) self.assertIs(type(s[1]), tuple) @@ -384,7 +397,7 @@ def test_setstate_subclasses(self): def test_recursive_pickle(self): with replaced_module('functools', self.module): f = self.partial(capture) - f.__setstate__((f, (), {}, 0, {})) + f.__setstate__((f, (), {}, {})) try: for proto in range(pickle.HIGHEST_PROTOCOL + 1): # gh-117008: Small limit since pickle uses C stack memory @@ -392,31 +405,31 @@ def test_recursive_pickle(self): with self.assertRaises(RecursionError): pickle.dumps(f, proto) finally: - f.__setstate__((capture, (), {}, 0, {})) + f.__setstate__((capture, (), {}, {})) f = self.partial(capture) - f.__setstate__((capture, (f,), {}, 0, {})) + f.__setstate__((capture, (f,), {}, {})) try: for proto in range(pickle.HIGHEST_PROTOCOL + 1): f_copy = pickle.loads(pickle.dumps(f, proto)) try: self.assertIs(f_copy.args[0], f_copy) finally: - f_copy.__setstate__((capture, (), {}, 0, {})) + f_copy.__setstate__((capture, (), {}, {})) finally: - f.__setstate__((capture, (), {}, 0, {})) + f.__setstate__((capture, (), {}, {})) f = self.partial(capture) - f.__setstate__((capture, (), {'a': f}, 0, {})) + f.__setstate__((capture, (), {'a': f}, {})) try: for proto in range(pickle.HIGHEST_PROTOCOL + 1): f_copy = pickle.loads(pickle.dumps(f, proto)) try: self.assertIs(f_copy.keywords['a'], f_copy) finally: - f_copy.__setstate__((capture, (), {}, 0, {})) + f_copy.__setstate__((capture, (), {}, {})) finally: - f.__setstate__((capture, (), {}, 0, {})) + f.__setstate__((capture, (), {}, {})) # Issue 6083: Reference counting bug def test_setstate_refcount(self): diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 1368648b5625f9..13b6d419af9f29 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -646,9 +646,25 @@ partial_setstate(partialobject *pto, PyObject *state) { PyObject *fn, *fnargs, *kw, *dict; Py_ssize_t np; - - if (!PyTuple_Check(state) || - !PyArg_ParseTuple(state, "OOOnO", &fn, &fnargs, &kw, &np, &dict) || + if (!PyTuple_Check(state)) { + PyErr_SetString(PyExc_TypeError, "invalid partial state"); + return NULL; + } + Py_ssize_t state_len = PyTuple_GET_SIZE(state); + int parse_rtrn; + if (state_len == 4) { + /* pre-placeholder support */ + parse_rtrn = PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict); + np = 0; + } + else if (state_len == 5) { + parse_rtrn = PyArg_ParseTuple(state, "OOOnO", &fn, &fnargs, &kw, &np, &dict); + } + else { + PyErr_SetString(PyExc_TypeError, "invalid partial state"); + return NULL; + } + if (!parse_rtrn || !PyCallable_Check(fn) || !PyTuple_Check(fnargs) || (kw != Py_None && !PyDict_Check(kw))) From 8ccc38f17304fb8556245f8ea07723500cacd304 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 20 Jun 2024 16:09:27 +0300 Subject: [PATCH 28/70] better variable names --- Lib/functools.py | 52 +++++----- Modules/_functoolsmodule.c | 200 +++++++++++++++++++------------------ 2 files changed, 127 insertions(+), 125 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 7bd962f7a7bab5..77d036dd5bd4e1 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -314,29 +314,29 @@ def __new__(cls, func, /, *args, **keywords): if args: if args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") - np = args.count(Placeholder) + phcount = args.count(Placeholder) else: - np = 0 + phcount = 0 if isinstance(func, partial): - pargs = func.args - pnp = func.placeholder_count + pto_args = func.args + pto_phcount = func.placeholder_count # merge args with args of `func` which is `partial` - if pnp and args: - all_args = list(pargs) + if pto_phcount and args: + tot_args = list(pto_args) nargs = len(args) pos = j = 0 - end = nargs if nargs < pnp else pnp + end = nargs if nargs < pto_phcount else pto_phcount while j < end: - pos = all_args.index(Placeholder, pos) - all_args[pos] = args[j] + pos = tot_args.index(Placeholder, pos) + tot_args[pos] = args[j] pos += 1 j += 1 - if pnp < nargs: - all_args.extend(args[pnp:]) - np += pnp - end - args = tuple(all_args) + if pto_phcount < nargs: + tot_args.extend(args[pto_phcount:]) + phcount += pto_phcount - end + args = tuple(tot_args) else: - np += pnp + phcount += pto_phcount args = func.args + args keywords = {**func.keywords, **keywords} func = func.func @@ -345,29 +345,29 @@ def __new__(cls, func, /, *args, **keywords): self.func = func self.args = args self.keywords = keywords - self.placeholder_count = np + self.placeholder_count = phcount return self def __call__(self, /, *args, **keywords): - np = self.placeholder_count - p_args = self.args - if np: + pto_phcount = self.placeholder_count + pto_args = self.args + if pto_phcount: n = len(args) - if n < np: + if n < pto_phcount: raise TypeError( "missing positional arguments " "in 'partial' call; expected " - f"at least {np}, got {n}") - p_args = list(p_args) + f"at least {pto_phcount}, got {n}") + pto_args = list(pto_args) pos = j = 0 - while j < np: - pos = p_args.index(Placeholder, pos) - p_args[pos] = args[j] + while j < pto_phcount: + pos = pto_args.index(Placeholder, pos) + pto_args[pos] = args[j] pos += 1 j += 1 - args = args[np:] if n > np else () + args = args[pto_phcount:] if n > pto_phcount else () keywords = {**self.keywords, **keywords} - return self.func(*p_args, *args, **keywords) + return self.func(*pto_args, *args, **keywords) @recursive_repr() def __repr__(self): diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 13b6d419af9f29..76edf2a7cc8099 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -120,7 +120,7 @@ typedef struct { PyObject *dict; /* __dict__ */ PyObject *weakreflist; /* List of weak references */ PyObject *placeholder; /* Placeholder for positional arguments */ - Py_ssize_t np; /* Number of placeholders */ + Py_ssize_t phcount; /* Number of placeholders */ vectorcallfunc vectorcall; } partialobject; @@ -143,17 +143,17 @@ get_functools_state_by_type(PyTypeObject *type) static PyObject * partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *func, *pargs, *nargs, *pkw; + PyObject *func, *pto_args, *new_args, *pto_kw; partialobject *pto; - Py_ssize_t pnp = 0; - Py_ssize_t nnargs = PyTuple_GET_SIZE(args); + Py_ssize_t pto_phcount; + Py_ssize_t new_nargs = PyTuple_GET_SIZE(args); - if (nnargs < 1) { + if (new_nargs < 1) { PyErr_SetString(PyExc_TypeError, "type 'partial' takes at least one argument"); return NULL; } - nnargs--; + new_nargs--; func = PyTuple_GET_ITEM(args, 0); if (!PyCallable_Check(func)) { PyErr_SetString(PyExc_TypeError, @@ -166,7 +166,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - pargs = pkw = NULL; + pto_args = pto_kw = NULL; int res = PyObject_TypeCheck(func, state->partial_type); if (res == -1) { return NULL; @@ -175,14 +175,17 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) // We can use its underlying function directly and merge the arguments. partialobject *part = (partialobject *)func; if (part->dict == NULL) { - pargs = part->args; - pkw = part->kw; + pto_args = part->args; + pto_kw = part->kw; func = part->fn; - pnp = part->np; - assert(PyTuple_Check(pargs)); - assert(PyDict_Check(pkw)); + pto_phcount = part->phcount; + assert(PyTuple_Check(pto_args)); + assert(PyDict_Check(pto_kw)); } } + else { + pto_phcount = 0; + } /* create partialobject structure */ pto = (partialobject *)type->tp_alloc(type, 0); @@ -192,64 +195,64 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) pto->fn = Py_NewRef(func); pto->placeholder = state->placeholder; - if (Py_Is(PyTuple_GET_ITEM(args, nnargs), pto->placeholder)) { + if (Py_Is(PyTuple_GET_ITEM(args, new_nargs), pto->placeholder)) { PyErr_SetString(PyExc_TypeError, "trailing Placeholders are not allowed"); return NULL; } - nargs = PyTuple_GetSlice(args, 1, nnargs + 1); - if (nargs == NULL) { + new_args = PyTuple_GetSlice(args, 1, new_nargs + 1); + if (new_args == NULL) { Py_DECREF(pto); return NULL; } - Py_ssize_t nnp = 0; + Py_ssize_t phcount = 0; PyObject *item; /* Count placeholders */ - if (nnargs > 1) { - for (Py_ssize_t i = 0; i < nnargs - 1; i++) { - item = PyTuple_GET_ITEM(nargs, i); + if (new_nargs > 1) { + for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { + item = PyTuple_GET_ITEM(new_args, i); if (Py_Is(item, pto->placeholder)) { - nnp++; + phcount++; } } } /* merge args with args of `func` which is `partial` */ - if ((pnp > 0) && (nnargs > 0)) { - Py_ssize_t npargs = PyTuple_GET_SIZE(pargs); - Py_ssize_t nfargs = npargs; - if (nnargs > pnp) - nfargs += nnargs - pnp; - PyObject *fargs = PyTuple_New(nfargs); - for (Py_ssize_t i = 0, j = 0; i < nfargs; i++) { + if ((pto_phcount > 0) && (new_nargs > 0)) { + Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); + Py_ssize_t tot_nargs = npargs; + if (new_nargs > pto_phcount) + tot_nargs += new_nargs - pto_phcount; + PyObject *tot_args = PyTuple_New(tot_nargs); + for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { if (i < npargs) { - item = PyTuple_GET_ITEM(pargs, i); - if ((j < nnargs) & Py_Is(item, pto->placeholder)) { - item = PyTuple_GET_ITEM(nargs, j); + item = PyTuple_GET_ITEM(pto_args, i); + if ((j < new_nargs) & Py_Is(item, pto->placeholder)) { + item = PyTuple_GET_ITEM(new_args, j); j++; - pnp--; + pto_phcount--; } } else { - item = PyTuple_GET_ITEM(nargs, j); + item = PyTuple_GET_ITEM(new_args, j); j++; } Py_INCREF(item); - PyTuple_SET_ITEM(fargs, i, item); + PyTuple_SET_ITEM(tot_args, i, item); } - pto->args = fargs; - pto->np = pnp + nnp; - Py_DECREF(nargs); + pto->args = tot_args; + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); } - else if (pargs == NULL) { - pto->args = nargs; - pto->np = nnp; + else if (pto_args == NULL) { + pto->args = new_args; + pto->phcount = phcount; } else { - pto->args = PySequence_Concat(pargs, nargs); - pto->np = pnp + nnp; - Py_DECREF(nargs); + pto->args = PySequence_Concat(pto_args, new_args); + pto->phcount = pto_phcount + phcount; + Py_DECREF(new_args); if (pto->args == NULL) { Py_DECREF(pto); return NULL; @@ -257,7 +260,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) assert(PyTuple_Check(pto->args)); } - if (pkw == NULL || PyDict_GET_SIZE(pkw) == 0) { + if (pto_kw == NULL || PyDict_GET_SIZE(pto_kw) == 0) { if (kw == NULL) { pto->kw = PyDict_New(); } @@ -269,7 +272,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } } else { - pto->kw = PyDict_Copy(pkw); + pto->kw = PyDict_Copy(pto_kw); if (kw != NULL && pto->kw != NULL) { if (PyDict_Merge(pto->kw, kw, 1) != 0) { Py_DECREF(pto); @@ -347,24 +350,24 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, if (PyDict_GET_SIZE(pto->kw)) { return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames); } - Py_ssize_t np = pto->np; - if (nargs < np) { + Py_ssize_t pto_phcount = pto->phcount; + if (nargs < pto_phcount) { PyErr_Format(PyExc_TypeError, "missing positional arguments in 'partial' call; " - "expected at least %zd, got %zd", np, nargs); + "expected at least %zd, got %zd", pto_phcount, nargs); return NULL; } - Py_ssize_t nargs_total = nargs; + Py_ssize_t nargskw = nargs; if (kwnames != NULL) { - nargs_total += PyTuple_GET_SIZE(kwnames); + nargskw += PyTuple_GET_SIZE(kwnames); } PyObject **pto_args = _PyTuple_ITEMS(pto->args); Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); /* Fast path if we're called without arguments */ - if (nargs_total == 0) { + if (nargskw == 0) { return _PyObject_VectorcallTstate(tstate, pto->fn, pto_args, pto_nargs, NULL); } @@ -381,49 +384,47 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, return ret; } - Py_ssize_t newnargs_total = pto_nargs + nargs_total; - newnargs_total = newnargs_total - np; - PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; - PyObject *ret; PyObject **stack; - if (newnargs_total <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + Py_ssize_t tot_nargskw = pto_nargs + nargskw - pto_phcount; + if (tot_nargskw <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { stack = small_stack; } else { - stack = PyMem_Malloc(newnargs_total * sizeof(PyObject *)); + stack = PyMem_Malloc(tot_nargskw * sizeof(PyObject *)); if (stack == NULL) { PyErr_NoMemory(); return NULL; } } - Py_ssize_t nargs_new; - if (np) { - nargs_new = pto_nargs + nargs - np; + Py_ssize_t tot_nargs; + if (pto_phcount) { + tot_nargs = pto_nargs + nargs - pto_phcount; Py_ssize_t j = 0; // New args index for (Py_ssize_t i = 0; i < pto_nargs; i++) { if (Py_Is(pto_args[i], pto->placeholder)){ - memcpy(stack + i, args + j, 1 * sizeof(PyObject*)); + stack[i] = args[j]; j += 1; } else { - memcpy(stack + i, pto_args + i, 1 * sizeof(PyObject*)); + stack[i] = pto_args[i]; } } - assert(j == np); - if (nargs_total > np) { - memcpy(stack + pto_nargs, args + j, (nargs_total - j) * sizeof(PyObject*)); + assert(j == pto_phcount); + if (nargskw > pto_phcount) { + memcpy(stack + pto_nargs, args + j, (nargskw - j) * sizeof(PyObject*)); } } else { - nargs_new = pto_nargs + nargs; + tot_nargs = pto_nargs + nargs; /* Copy to new stack, using borrowed references */ memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); - memcpy(stack + pto_nargs, args, nargs_total * sizeof(PyObject*)); + memcpy(stack + pto_nargs, args, nargskw * sizeof(PyObject*)); } - ret = _PyObject_VectorcallTstate(tstate, pto->fn, stack, nargs_new, kwnames); + PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, + stack, tot_nargs, kwnames); if (stack != small_stack) { PyMem_Free(stack); } @@ -456,45 +457,45 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) assert(PyDict_Check(pto->kw)); Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t np = pto->np; - if (nargs < np) { + Py_ssize_t pto_phcount = pto->phcount; + if (nargs < pto_phcount) { PyErr_Format(PyExc_TypeError, "missing positional arguments in 'partial' call; " - "expected at least %zd, got %zd", np, nargs); + "expected at least %zd, got %zd", pto_phcount, nargs); return NULL; } /* Merge keywords */ - PyObject *kwargs2; + PyObject *tot_kw; if (PyDict_GET_SIZE(pto->kw) == 0) { /* kwargs can be NULL */ - kwargs2 = Py_XNewRef(kwargs); + tot_kw = Py_XNewRef(kwargs); } else { /* bpo-27840, bpo-29318: dictionary of keyword parameters must be copied, because a function using "**kwargs" can modify the dictionary. */ - kwargs2 = PyDict_Copy(pto->kw); - if (kwargs2 == NULL) { + tot_kw = PyDict_Copy(pto->kw); + if (tot_kw == NULL) { return NULL; } if (kwargs != NULL) { - if (PyDict_Merge(kwargs2, kwargs, 1) != 0) { - Py_DECREF(kwargs2); + if (PyDict_Merge(tot_kw, kwargs, 1) != 0) { + Py_DECREF(tot_kw); return NULL; } } } /* Merge positional arguments */ - PyObject *args2; - if (np) { + PyObject *tot_args; + if (pto_phcount) { Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); - Py_ssize_t nargs_new = pto_nargs + nargs - np; - args2 = PyTuple_New(nargs_new); - if (args2 == NULL) { - Py_XDECREF(kwargs2); + Py_ssize_t tot_nargs = pto_nargs + nargs - pto_phcount; + tot_args = PyTuple_New(tot_nargs); + if (tot_args == NULL) { + Py_XDECREF(tot_kw); return NULL; } PyObject *pto_args = pto->args; @@ -506,29 +507,29 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) item = PyTuple_GET_ITEM(args, j); j += 1; } - PyTuple_SET_ITEM(args2, i, item); + PyTuple_SET_ITEM(tot_args, i, item); } - assert(j == np); - if (nargs > np) { - for (Py_ssize_t i = pto_nargs; i < nargs_new; i++) { + assert(j == pto_phcount); + if (nargs > pto_phcount) { + for (Py_ssize_t i = pto_nargs; i < tot_nargs; i++) { item = PyTuple_GET_ITEM(args, j); - PyTuple_SET_ITEM(args2, i, item); + PyTuple_SET_ITEM(tot_args, i, item); j += 1; } } } else { /* Note: tupleconcat() is optimized for empty tuples */ - args2 = PySequence_Concat(pto->args, args); - if (args2 == NULL) { - Py_XDECREF(kwargs2); + tot_args = PySequence_Concat(pto->args, args); + if (tot_args == NULL) { + Py_XDECREF(tot_kw); return NULL; } } - PyObject *res = PyObject_Call(pto->fn, args2, kwargs2); - Py_DECREF(args2); - Py_XDECREF(kwargs2); + PyObject *res = PyObject_Call(pto->fn, tot_args, tot_kw); + Py_DECREF(tot_args); + Py_XDECREF(tot_kw); return res; } @@ -545,7 +546,7 @@ static PyMemberDef partial_memberlist[] = { "tuple of arguments to future partial calls"}, {"keywords", _Py_T_OBJECT, OFF(kw), Py_READONLY, "dictionary of keyword arguments to future partial calls"}, - {"placeholder_count", Py_T_PYSSIZET, OFF(np), Py_READONLY, + {"placeholder_count", Py_T_PYSSIZET, OFF(phcount), Py_READONLY, "number of placeholders"}, {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(partialobject, weakreflist), Py_READONLY}, @@ -637,7 +638,7 @@ static PyObject * partial_reduce(partialobject *pto, PyObject *unused) { return Py_BuildValue("O(O)(OOOnO)", Py_TYPE(pto), pto->fn, pto->fn, - pto->args, pto->kw, pto->np, + pto->args, pto->kw, pto->phcount, pto->dict ? pto->dict : Py_None); } @@ -645,7 +646,7 @@ static PyObject * partial_setstate(partialobject *pto, PyObject *state) { PyObject *fn, *fnargs, *kw, *dict; - Py_ssize_t np; + Py_ssize_t phcount; if (!PyTuple_Check(state)) { PyErr_SetString(PyExc_TypeError, "invalid partial state"); return NULL; @@ -655,10 +656,11 @@ partial_setstate(partialobject *pto, PyObject *state) if (state_len == 4) { /* pre-placeholder support */ parse_rtrn = PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict); - np = 0; + phcount = 0; } else if (state_len == 5) { - parse_rtrn = PyArg_ParseTuple(state, "OOOnO", &fn, &fnargs, &kw, &np, &dict); + parse_rtrn = PyArg_ParseTuple(state, "OOOnO", &fn, &fnargs, + &kw, &phcount, &dict); } else { PyErr_SetString(PyExc_TypeError, "invalid partial state"); @@ -698,7 +700,7 @@ partial_setstate(partialobject *pto, PyObject *state) Py_SETREF(pto->fn, Py_NewRef(fn)); Py_SETREF(pto->args, fnargs); Py_SETREF(pto->kw, kw); - pto->np = np; + pto->phcount = phcount; Py_XSETREF(pto->dict, dict); partial_setvectorcall(pto); Py_RETURN_NONE; From e7c82c7a3dbc62ae8d531883a6698127ebce0bfa Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 20 Jun 2024 17:35:20 +0300 Subject: [PATCH 29/70] partialmethod impl --- Lib/functools.py | 209 +++++++++++++++++++------------------ Modules/_functoolsmodule.c | 3 +- 2 files changed, 109 insertions(+), 103 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 77d036dd5bd4e1..97d5661cd46ea8 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -299,6 +299,71 @@ def __reduce__(self): Placeholder = type('PlaceholderType', (__PlaceholderTypeBase,), {})() +def _partial_prepare_new(cls, func, args, keywords): + if args: + if args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + phcount = args.count(Placeholder) + else: + phcount = 0 + if isinstance(func, cls): + pto_args = func.args + pto_phcount = func.placeholder_count + # merge args with args of `func` which is `partial` + if pto_phcount and args: + tot_args = list(pto_args) + nargs = len(args) + pos = j = 0 + end = nargs if nargs < pto_phcount else pto_phcount + while j < end: + pos = tot_args.index(Placeholder, pos) + tot_args[pos] = args[j] + pos += 1 + j += 1 + if pto_phcount < nargs: + tot_args.extend(args[pto_phcount:]) + phcount += pto_phcount - end + args = tuple(tot_args) + else: + phcount += pto_phcount + args = func.args + args + keywords = {**func.keywords, **keywords} + func = func.func + return func, args, keywords, phcount + + +def _partial_prepare_call(self, args, keywords): + pto_phcount = self.placeholder_count + pto_args = self.args + if pto_phcount: + n = len(args) + if n < pto_phcount: + raise TypeError( + "missing positional arguments " + "in 'partial' call; expected " + f"at least {pto_phcount}, got {n}") + pto_args = list(pto_args) + pos = j = 0 + while j < pto_phcount: + pos = pto_args.index(Placeholder, pos) + pto_args[pos] = args[j] + pos += 1 + j += 1 + args = args[pto_phcount:] if n > pto_phcount else () + keywords = {**self.keywords, **keywords} + return pto_args, args, keywords + + +def _partial_repr(self): + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" + + # Purely functional, no descriptor behaviour class partial: """New function with partial application of the given arguments @@ -311,96 +376,48 @@ class partial: def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") - if args: - if args[-1] is Placeholder: - raise TypeError("trailing Placeholders are not allowed") - phcount = args.count(Placeholder) - else: - phcount = 0 - if isinstance(func, partial): - pto_args = func.args - pto_phcount = func.placeholder_count - # merge args with args of `func` which is `partial` - if pto_phcount and args: - tot_args = list(pto_args) - nargs = len(args) - pos = j = 0 - end = nargs if nargs < pto_phcount else pto_phcount - while j < end: - pos = tot_args.index(Placeholder, pos) - tot_args[pos] = args[j] - pos += 1 - j += 1 - if pto_phcount < nargs: - tot_args.extend(args[pto_phcount:]) - phcount += pto_phcount - end - args = tuple(tot_args) - else: - phcount += pto_phcount - args = func.args + args - keywords = {**func.keywords, **keywords} - func = func.func - - self = super(partial, cls).__new__(cls) + func, args, kwds, phcount = _partial_prepare_new(cls, func, args, + keywords) + self = super().__new__(cls) self.func = func self.args = args - self.keywords = keywords + self.keywords = kwds self.placeholder_count = phcount return self def __call__(self, /, *args, **keywords): - pto_phcount = self.placeholder_count - pto_args = self.args - if pto_phcount: - n = len(args) - if n < pto_phcount: - raise TypeError( - "missing positional arguments " - "in 'partial' call; expected " - f"at least {pto_phcount}, got {n}") - pto_args = list(pto_args) - pos = j = 0 - while j < pto_phcount: - pos = pto_args.index(Placeholder, pos) - pto_args[pos] = args[j] - pos += 1 - j += 1 - args = args[pto_phcount:] if n > pto_phcount else () - keywords = {**self.keywords, **keywords} - return self.func(*pto_args, *args, **keywords) + pargs, args, kwds = _partial_prepare_call(self, args, keywords) + return self.func(*pargs, *args, **kwds) - @recursive_repr() - def __repr__(self): - cls = type(self) - qualname = cls.__qualname__ - module = cls.__module__ - args = [repr(self.func)] - args.extend(repr(x) for x in self.args) - args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - return f"{module}.{qualname}({', '.join(args)})" + __repr__ = recursive_repr()(_partial_repr) def __reduce__(self): - return type(self), (self.func,), (self.func, self.args, - self.keywords or None, self.placeholder_count, - self.__dict__ or None) + state = ( + self.func, + self.args, + self.keywords or None, + self.placeholder_count, + self.__dict__ or None + ) + return type(self), (self.func,), state def __setstate__(self, state): if not isinstance(state, tuple): raise TypeError("argument to __setstate__ must be a tuple") - n = len(state) - if n == 4: + state_len = len(state) + if state_len == 4: # Support pre-placeholder de-serialization func, args, kwds, namespace = state - placeholder_count = 0 - elif n == 5: - func, args, kwds, placeholder_count, namespace = state + phcount = 0 + elif state_len == 5: + func, args, kwds, phcount, namespace = state else: - raise TypeError(f"expected 5 items in state, got {len(state)}") + raise TypeError(f"expected 4 or 5 items in state, got {state_len}") if (not callable(func) or not isinstance(args, tuple) or - (kwds is not None and not isinstance(kwds, dict)) or - not isinstance(placeholder_count, int) or - (namespace is not None and not isinstance(namespace, dict))): + (kwds is not None and not isinstance(kwds, dict)) or + not isinstance(phcount, int) or + (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") args = tuple(args) # just in case it's a subclass @@ -415,7 +432,7 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds - self.placeholder_count = placeholder_count + self.placeholder_count = phcount try: from _functools import partial, Placeholder @@ -423,48 +440,36 @@ def __setstate__(self, state): pass # Descriptor version -class partialmethod(object): +class partialmethod: """Method descriptor with partial application of the given arguments and keywords. Supports wrapping existing descriptors and handles non-descriptor callables as instance methods. """ - - def __init__(self, func, /, *args, **keywords): + def __new__(cls, func, /, *args, **keywords): if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) - + raise TypeError(f"{func!r} is not callable or a descriptor") # func could be a descriptor like classmethod which isn't callable, # so we can't inherit from partial (it verifies func is callable) - if isinstance(func, partialmethod): - # flattening is mandatory in order to place cls/self before all - # other arguments - # it's also more efficient since only one function will be called - self.func = func.func - self.args = func.args + args - self.keywords = {**func.keywords, **keywords} - else: - self.func = func - self.args = args - self.keywords = keywords + # flattening is mandatory in order to place cls/self before all + # other arguments + # it's also more efficient since only one function will be called + func, args, kwds, phcount = _partial_prepare_new(cls, func, args, + keywords) + self = super().__new__(cls) + self.func = func + self.args = args + self.keywords = kwds + self.placeholder_count = phcount + return self - def __repr__(self): - args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) - format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + __repr__ = _partial_repr def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): - keywords = {**self.keywords, **keywords} - return self.func(cls_or_self, *self.args, *args, **keywords) + pargs, args, kwds = _partial_prepare_call(self, args, keywords) + return self.func(cls_or_self, *pargs, *args, **kwds) _method.__isabstractmethod__ = self.__isabstractmethod__ _method.__partialmethod__ = self return _method diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 76edf2a7cc8099..072d399b6ca390 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -663,7 +663,8 @@ partial_setstate(partialobject *pto, PyObject *state) &kw, &phcount, &dict); } else { - PyErr_SetString(PyExc_TypeError, "invalid partial state"); + PyErr_Format(PyExc_TypeError, + "expected 4 or 5 items in state, got %zd", state_len); return NULL; } if (!parse_rtrn || From c9b7ef35c731b3ead1247c17fc02a1ea7a7b0d1e Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 20 Jun 2024 17:45:32 +0300 Subject: [PATCH 30/70] fix tests --- Lib/test/test_asyncio/test_events.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 88c85a36b5d448..05406b620c511b 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2363,8 +2363,8 @@ def test_handle_repr(self): filename, lineno = test_utils.get_function_source(method) h = asyncio.Handle(cb, (), self.loop) - cb_regex = r'' - cb_regex = fr'functools.partialmethod\({cb_regex}, , \)\(\)' + cb_regex = r'' + cb_regex = fr'functools.partialmethod\({cb_regex}\)\(\)' regex = fr'^$' self.assertRegex(repr(h), regex) From e59d711b8cd6d34d3e1478c964a68096de06f734 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 20 Jun 2024 20:20:53 +0300 Subject: [PATCH 31/70] adjust inspect to partial Placeholders --- Lib/inspect.py | 8 +++++-- Lib/test/test_inspect/test_inspect.py | 31 +++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index e6e49a4ffa673a..b037b2e3e09d5f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2028,7 +2028,9 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): if param.kind is _POSITIONAL_ONLY: # If positional-only parameter is bound by partial, # it effectively disappears from the signature - new_params.pop(param_name) + # However, if it is a Placeholder it is not removed + if arg_value != functools.Placeholder: + new_params.pop(param_name) continue if param.kind is _POSITIONAL_OR_KEYWORD: @@ -2050,7 +2052,9 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): new_params[param_name] = param.replace(default=arg_value) else: # was passed as a positional argument - new_params.pop(param.name) + # But do not remove if it is a Placeholder + if arg_value != functools.Placeholder: + new_params.pop(param.name) continue if param.kind is _KEYWORD_ONLY: diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 011d42f34b6461..26455bab396ac7 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3254,7 +3254,7 @@ def foo(cls, *, arg): ...)) def test_signature_on_partial(self): - from functools import partial + from functools import partial, Placeholder def test(): pass @@ -3309,6 +3309,19 @@ def test(a, b, *, c, d): ('d', ..., ..., "keyword_only")), ...)) + # With Placeholder + self.assertEqual(self.signature(partial(test, Placeholder, 1)), + ((('a', ..., ..., "positional_or_keyword"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, Placeholder, 1, c=2)), + ((('a', ..., ..., "positional_or_keyword"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + def test(a, *args, b, **kwargs): pass @@ -3472,17 +3485,31 @@ def test(): inspect.signature(Spam.ham) class Spam: - def test(it, a, *, c) -> 'spam': + def test(it, a, b, *, c) -> 'spam': pass ham = partialmethod(test, c=1) + bar = partialmethod(test, functools.Placeholder, 1, c=1) self.assertEqual(self.signature(Spam.ham, eval_str=False), ((('it', ..., ..., 'positional_or_keyword'), ('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), ('c', 1, ..., 'keyword_only')), 'spam')) self.assertEqual(self.signature(Spam().ham, eval_str=False), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + # With Placeholder + self.assertEqual(self.signature(Spam.bar, eval_str=False), + ((('it', ..., ..., 'positional_or_keyword'), + ('a', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + self.assertEqual(self.signature(Spam().bar, eval_str=False), ((('a', ..., ..., 'positional_or_keyword'), ('c', 1, ..., 'keyword_only')), 'spam')) From 7bfc59192f1b8811b5fd15edf1b0a1bbea4252b0 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 20 Jun 2024 20:44:30 +0300 Subject: [PATCH 32/70] arg alignment --- Lib/functools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/functools.py b/Lib/functools.py index 97d5661cd46ea8..91ffef3032a8c0 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -377,7 +377,7 @@ def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") func, args, kwds, phcount = _partial_prepare_new(cls, func, args, - keywords) + keywords) self = super().__new__(cls) self.func = func self.args = args From 7957a9700fffcd717555eb3a1d4ba0ddcfd7fecb Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Fri, 21 Jun 2024 13:02:52 +0300 Subject: [PATCH 33/70] small fixes --- Modules/_functoolsmodule.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 072d399b6ca390..681fa7d878a90f 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -145,7 +145,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *func, *pto_args, *new_args, *pto_kw; partialobject *pto; - Py_ssize_t pto_phcount; + Py_ssize_t pto_phcount = 0; Py_ssize_t new_nargs = PyTuple_GET_SIZE(args); if (new_nargs < 1) { @@ -183,9 +183,6 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) assert(PyDict_Check(pto_kw)); } } - else { - pto_phcount = 0; - } /* create partialobject structure */ pto = (partialobject *)type->tp_alloc(type, 0); @@ -222,13 +219,14 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if ((pto_phcount > 0) && (new_nargs > 0)) { Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); Py_ssize_t tot_nargs = npargs; - if (new_nargs > pto_phcount) + if (new_nargs > pto_phcount) { tot_nargs += new_nargs - pto_phcount; + } PyObject *tot_args = PyTuple_New(tot_nargs); for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { if (i < npargs) { item = PyTuple_GET_ITEM(pto_args, i); - if ((j < new_nargs) & Py_Is(item, pto->placeholder)) { + if ((j < new_nargs) && Py_Is(item, pto->placeholder)) { item = PyTuple_GET_ITEM(new_args, j); j++; pto_phcount--; From 8aaee6a0a854f99c463032b9138e80e2882046d8 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sun, 23 Jun 2024 19:14:38 +0300 Subject: [PATCH 34/70] pickle compatibility ensured --- Lib/functools.py | 26 +++++-------------- Lib/test/test_functools.py | 6 +---- Modules/_functoolsmodule.c | 51 +++++++++++++++++++------------------- 3 files changed, 33 insertions(+), 50 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 91ffef3032a8c0..6dc7e233a71002 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -392,31 +392,17 @@ def __call__(self, /, *args, **keywords): __repr__ = recursive_repr()(_partial_repr) def __reduce__(self): - state = ( - self.func, - self.args, - self.keywords or None, - self.placeholder_count, - self.__dict__ or None - ) - return type(self), (self.func,), state + return type(self), (self.func,), (self.func, self.args, + self.keywords or None, self.__dict__ or None) def __setstate__(self, state): if not isinstance(state, tuple): raise TypeError("argument to __setstate__ must be a tuple") - state_len = len(state) - if state_len == 4: - # Support pre-placeholder de-serialization - func, args, kwds, namespace = state - phcount = 0 - elif state_len == 5: - func, args, kwds, phcount, namespace = state - else: - raise TypeError(f"expected 4 or 5 items in state, got {state_len}") - + if len(state) != 4: + raise TypeError(f"expected 4 items in state, got {len(state)}") + func, args, kwds, namespace = state if (not callable(func) or not isinstance(args, tuple) or (kwds is not None and not isinstance(kwds, dict)) or - not isinstance(phcount, int) or (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") @@ -432,7 +418,7 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds - self.placeholder_count = phcount + self.placeholder_count = args.count(Placeholder) try: from _functools import partial, Placeholder diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index c401f30b996015..a90a93e84f142c 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -352,13 +352,9 @@ def test_setstate(self): self.assertEqual(f(), ((), {})) # Set State with placeholders - f = self.partial(signature) - f.__setstate__((capture, (1,), dict(a=10), 0, dict(attr=[]))) - self.assertEqual(signature(f), (capture, (1,), dict(a=10), dict(attr=[]))) - PH = self.module.Placeholder f = self.partial(signature) - f.__setstate__((capture, (PH, 1), dict(a=10), 1, dict(attr=[]))) + f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[]))) self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[]))) with self.assertRaises(TypeError): self.assertEqual(f(), (PH, 1), dict(a=10)) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 681fa7d878a90f..30afa6300b615a 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -192,7 +192,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) pto->fn = Py_NewRef(func); pto->placeholder = state->placeholder; - if (Py_Is(PyTuple_GET_ITEM(args, new_nargs), pto->placeholder)) { + if (new_nargs && Py_Is(PyTuple_GET_ITEM(args, new_nargs), pto->placeholder)) { PyErr_SetString(PyExc_TypeError, "trailing Placeholders are not allowed"); return NULL; @@ -204,15 +204,11 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - Py_ssize_t phcount = 0; - PyObject *item; /* Count placeholders */ - if (new_nargs > 1) { - for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { - item = PyTuple_GET_ITEM(new_args, i); - if (Py_Is(item, pto->placeholder)) { - phcount++; - } + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { + if (Py_Is(PyTuple_GET_ITEM(new_args, i), pto->placeholder)) { + phcount++; } } /* merge args with args of `func` which is `partial` */ @@ -222,6 +218,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (new_nargs > pto_phcount) { tot_nargs += new_nargs - pto_phcount; } + PyObject *item; PyObject *tot_args = PyTuple_New(tot_nargs); for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { if (i < npargs) { @@ -635,8 +632,8 @@ partial_repr(partialobject *pto) static PyObject * partial_reduce(partialobject *pto, PyObject *unused) { - return Py_BuildValue("O(O)(OOOnO)", Py_TYPE(pto), pto->fn, pto->fn, - pto->args, pto->kw, pto->phcount, + return Py_BuildValue("O(O)(OOOO)", Py_TYPE(pto), pto->fn, pto->fn, + pto->args, pto->kw, pto->dict ? pto->dict : Py_None); } @@ -644,28 +641,18 @@ static PyObject * partial_setstate(partialobject *pto, PyObject *state) { PyObject *fn, *fnargs, *kw, *dict; - Py_ssize_t phcount; + if (!PyTuple_Check(state)) { PyErr_SetString(PyExc_TypeError, "invalid partial state"); return NULL; } Py_ssize_t state_len = PyTuple_GET_SIZE(state); - int parse_rtrn; - if (state_len == 4) { - /* pre-placeholder support */ - parse_rtrn = PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict); - phcount = 0; - } - else if (state_len == 5) { - parse_rtrn = PyArg_ParseTuple(state, "OOOnO", &fn, &fnargs, - &kw, &phcount, &dict); - } - else { + if (state_len != 4) { PyErr_Format(PyExc_TypeError, - "expected 4 or 5 items in state, got %zd", state_len); + "expected 4 items in state, got %zd", state_len); return NULL; } - if (!parse_rtrn || + if (!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || !PyCallable_Check(fn) || !PyTuple_Check(fnargs) || (kw != Py_None && !PyDict_Check(kw))) @@ -674,6 +661,20 @@ partial_setstate(partialobject *pto, PyObject *state) return NULL; } + Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs); + if (nargs && Py_Is(PyTuple_GET_ITEM(fnargs, nargs - 1), pto->placeholder)) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + /* Count placeholders */ + Py_ssize_t phcount = 0; + for (Py_ssize_t i = 0; i < nargs - 1; i++) { + if (Py_Is(PyTuple_GET_ITEM(fnargs, i), pto->placeholder)) { + phcount++; + } + } + if(!PyTuple_CheckExact(fnargs)) fnargs = PySequence_Tuple(fnargs); else From fe8e0ad3f757afae50c0a1b3cff448d12acfcc99 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Sun, 23 Jun 2024 19:18:42 +0300 Subject: [PATCH 35/70] trailing placeholder test for __setstate__ --- Lib/functools.py | 9 ++++++++- Lib/test/test_functools.py | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/functools.py b/Lib/functools.py index 6dc7e233a71002..999c07a06445e2 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -406,6 +406,13 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") + if args: + if args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + phcount = args.count(Placeholder) + else: + phcount = 0 + args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} @@ -418,7 +425,7 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds - self.placeholder_count = args.count(Placeholder) + self.placeholder_count = phcount try: from _functools import partial, Placeholder diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index a90a93e84f142c..7bdda1cba05f33 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -360,6 +360,11 @@ def test_setstate(self): self.assertEqual(f(), (PH, 1), dict(a=10)) self.assertEqual(f(2), ((2, 1), dict(a=10))) + # Leading Placeholder error + f = self.partial(signature) + with self.assertRaises(TypeError): + f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) + def test_setstate_errors(self): f = self.partial(signature) self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) From 00dd80ed475573c0a430720660fbfd61ec16df1d Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 24 Jun 2024 09:40:48 +0300 Subject: [PATCH 36/70] rough implementation rolled back and example of successive applications added --- Doc/library/functools.rst | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 10f43edcfa3e5f..f03498f08afbea 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -340,30 +340,12 @@ The :mod:`functools` module defines the following functions: Placeholder = object() def partial(func, /, *args, **keywords): - # Trim trailing placeholders and count how many remaining - nargs = len(args) - while nargs and args[nargs-1] is Placeholder: - nargs -= 1 - args = args[:nargs] - placeholder_count = args.count(Placeholder) def newfunc(*fargs, **fkeywords): - if len(fargs) < placeholder_count: - raise TypeError("missing positional arguments in 'partial' call;" - f" expected at least {placeholder_count}, " - f"got {len(fargs)}") - newargs = list(args) - j = 0 - for i, arg in enumarate(args): - if arg is Placeholder: - newargs[i] = fargs[j] - j += 1 - newargs.extend(fargs[j:]) newkeywords = {**keywords, **fkeywords} - return func(*newargs, **newkeywords) + return func(*args, *fargs, **newkeywords) newfunc.func = func newfunc.args = args newfunc.keywords = keywords - newfunc.placeholder_count = placeholder_count return newfunc The :func:`partial` is used for partial function application which "freezes" @@ -391,7 +373,19 @@ The :mod:`functools` module defines the following functions: >>> say_to_world('Hello', 'dear') Hello dear world! - Calling ``say_to_world('Hello')`` would raise a :exc:`TypeError`. + Calling ``say_to_world('Hello')`` would raise a :exc:`TypeError`, because + only one positional argument is provided, while there are two placeholders + in :ref:`partial object `. + + Placeholders can be used repetitively to reduce the number + of positional arguments with each successive application: + + >>> from functools import partial, Placeholder as _ + >>> count = partial(print, _, 2, _, _, 5) + >>> count = partial(count, 1, _, 4) + >>> count = partial(count, 3) + >>> count() + 1 2 3 4 5 .. versionchanged:: 3.14 Support for :data:`Placeholder` in *args* From d352cfafb297105fd1cab3db83dc26f0286a5e3f Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 24 Jun 2024 09:53:45 +0300 Subject: [PATCH 37/70] small doc edits --- Doc/library/functools.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index f03498f08afbea..61832b1258f56c 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -377,15 +377,15 @@ The :mod:`functools` module defines the following functions: only one positional argument is provided, while there are two placeholders in :ref:`partial object `. - Placeholders can be used repetitively to reduce the number - of positional arguments with each successive application: + :data:`!Placeholder` sentinels can be used repetitively to reduce the number + of positional arguments with each successive application. A place for positional + argument is retained when :data:`!Placeholder` sentinel is replaced with the new one: >>> from functools import partial, Placeholder as _ - >>> count = partial(print, _, 2, _, _, 5) - >>> count = partial(count, 1, _, 4) - >>> count = partial(count, 3) - >>> count() - 1 2 3 4 5 + >>> count = partial(print, _, _, _, 4) + >>> count = partial(count, _, 2) + >>> count(1, 3) + 1 2 3 4 .. versionchanged:: 3.14 Support for :data:`Placeholder` in *args* From 9038ed5779523974133806d27666eb240631a02b Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 24 Jun 2024 10:41:47 +0300 Subject: [PATCH 38/70] example changes --- Doc/library/functools.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 61832b1258f56c..ee5db4bf1f10b0 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -377,15 +377,17 @@ The :mod:`functools` module defines the following functions: only one positional argument is provided, while there are two placeholders in :ref:`partial object `. - :data:`!Placeholder` sentinels can be used repetitively to reduce the number - of positional arguments with each successive application. A place for positional - argument is retained when :data:`!Placeholder` sentinel is replaced with the new one: + When successively using :func:`partial` existing :data:`!Placeholder` + sentinels are filled first. A place for positional argument is retained + when :data:`!Placeholder` sentinel is replaced with a new one: >>> from functools import partial, Placeholder as _ >>> count = partial(print, _, _, _, 4) + >>> count = partial(count, _, _, 3) >>> count = partial(count, _, 2) - >>> count(1, 3) - 1 2 3 4 + >>> count = partial(count, _, 5) # 5 is appended after 4 + >>> count(1) + 1 2 3 4 5 .. versionchanged:: 3.14 Support for :data:`Placeholder` in *args* From 49b8c719957efae75a72c8472d95e84e5425f16d Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 25 Jun 2024 16:43:56 +0300 Subject: [PATCH 39/70] serialization issues addressed --- Lib/functools.py | 9 ++++++++- Lib/pickle.py | 4 +++- Modules/_functoolsmodule.c | 12 ++++++++++++ Modules/_pickle.c | 9 +++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 999c07a06445e2..7dafe3be0c47c2 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -280,6 +280,7 @@ class __PlaceholderTypeBase: Used as a placeholder for partial arguments. """ __instance = None + __slots__ = () def __new__(cls): if cls.__instance is None: @@ -295,8 +296,14 @@ def __repr__(self): def __reduce__(self): return 'Placeholder' +def __placeholder_init_subclass__(cls, *args, **kwargs): + raise TypeError(f"type '{cls.__name__}' is not an acceptable base type") -Placeholder = type('PlaceholderType', (__PlaceholderTypeBase,), {})() +Placeholder = type( + 'PlaceholderType', + (__PlaceholderTypeBase,), + {'__slots__': (), '__init_subclass__': __placeholder_init_subclass__} +)() def _partial_prepare_new(cls, func, args, keywords): diff --git a/Lib/pickle.py b/Lib/pickle.py index 33c97c8c5efb28..62d196813bd847 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -27,7 +27,7 @@ from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache from itertools import islice -from functools import partial +from functools import partial, Placeholder import sys from sys import maxsize from struct import pack, unpack @@ -1120,6 +1120,8 @@ def save_type(self, obj): return self.save_reduce(type, (NotImplemented,), obj=obj) elif obj is type(...): return self.save_reduce(type, (...,), obj=obj) + elif obj is type(Placeholder): + return self.save_reduce(type, (Placeholder,), obj=obj) return self.save_global(obj) dispatch[FunctionType] = save_global diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 30afa6300b615a..ce3fde52cff383 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -63,6 +63,17 @@ placeholder_repr(PyObject *op) return PyUnicode_FromString("Placeholder"); } +static PyObject * +placeholder_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("Placeholder"); +} + +static PyMethodDef placeholder_methods[] = { + {"__reduce__", placeholder_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + static void placeholder_dealloc(PyObject* placeholder) { @@ -98,6 +109,7 @@ static PyType_Slot placeholder_type_slots[] = { {Py_tp_dealloc, placeholder_dealloc}, {Py_tp_repr, placeholder_repr}, {Py_tp_doc, (void *)placeholder_doc}, + {Py_tp_methods, placeholder_methods}, {Py_tp_new, placeholder_new}, // Number protocol {Py_nb_bool, placeholder_bool}, diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 754a326822e0f0..ff88f875235ed7 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -198,6 +198,7 @@ typedef struct { /* functools.partial, used for implementing __newobj_ex__ with protocols 2 and 3 */ PyObject *partial; + PyObject *placeholder; /* Types */ PyTypeObject *Pickler_Type; @@ -253,6 +254,7 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->codecs_encode); Py_CLEAR(st->getattr); Py_CLEAR(st->partial); + Py_CLEAR(st->placeholder); Py_CLEAR(st->Pickler_Type); Py_CLEAR(st->Unpickler_Type); Py_CLEAR(st->Pdata_Type); @@ -375,6 +377,9 @@ _Pickle_InitState(PickleState *st) st->partial = _PyImport_GetModuleAttrString("functools", "partial"); if (!st->partial) goto error; + st->placeholder = _PyImport_GetModuleAttrString("functools", "Placeholder"); + if (!st->placeholder) + goto error; return 0; @@ -3861,6 +3866,9 @@ save_type(PickleState *state, PicklerObject *self, PyObject *obj) else if (obj == (PyObject *)&_PyNotImplemented_Type) { return save_singleton_type(state, self, obj, Py_NotImplemented); } + else if (obj == (PyObject *)Py_TYPE(state->placeholder)) { + return save_singleton_type(state, self, obj, state->placeholder); + } return save_global(state, self, obj, NULL); } @@ -7791,6 +7799,7 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->codecs_encode); Py_VISIT(st->getattr); Py_VISIT(st->partial); + Py_VISIT(st->placeholder); Py_VISIT(st->Pickler_Type); Py_VISIT(st->Unpickler_Type); Py_VISIT(st->Pdata_Type); From bc1fdbd9d90a9bad3a1bc62254da94c4244ea7bb Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 25 Jun 2024 17:37:14 +0300 Subject: [PATCH 40/70] delete redundant references --- Lib/functools.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 7dafe3be0c47c2..968e67b69d8670 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -274,7 +274,7 @@ def reduce(function, sequence, initial=_initial_missing): ################################################################################ -class __PlaceholderTypeBase: +class PlaceholderTypeBase: """The type of the Placeholder singleton. Used as a placeholder for partial arguments. @@ -296,15 +296,16 @@ def __repr__(self): def __reduce__(self): return 'Placeholder' -def __placeholder_init_subclass__(cls, *args, **kwargs): +def placeholder_init_subclass(cls, *args, **kwargs): raise TypeError(f"type '{cls.__name__}' is not an acceptable base type") Placeholder = type( 'PlaceholderType', - (__PlaceholderTypeBase,), - {'__slots__': (), '__init_subclass__': __placeholder_init_subclass__} + (PlaceholderTypeBase,), + {'__slots__': (), '__init_subclass__': placeholder_init_subclass} )() +del PlaceholderTypeBase, placeholder_init_subclass def _partial_prepare_new(cls, func, args, keywords): if args: @@ -338,7 +339,6 @@ def _partial_prepare_new(cls, func, args, keywords): func = func.func return func, args, keywords, phcount - def _partial_prepare_call(self, args, keywords): pto_phcount = self.placeholder_count pto_args = self.args @@ -360,7 +360,6 @@ def _partial_prepare_call(self, args, keywords): keywords = {**self.keywords, **keywords} return pto_args, args, keywords - def _partial_repr(self): cls = type(self) module = cls.__module__ @@ -370,7 +369,6 @@ def _partial_repr(self): args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) return f"{module}.{qualname}({', '.join(args)})" - # Purely functional, no descriptor behaviour class partial: """New function with partial application of the given arguments From 3067221f1e012482fbe030a5de06c6e70c1b7b41 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 25 Jun 2024 17:48:24 +0300 Subject: [PATCH 41/70] simplify PyPlaceholder implementation --- Lib/functools.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 968e67b69d8670..40637954542fea 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -274,7 +274,7 @@ def reduce(function, sequence, initial=_initial_missing): ################################################################################ -class PlaceholderTypeBase: +class PlaceholderType: """The type of the Placeholder singleton. Used as a placeholder for partial arguments. @@ -282,6 +282,9 @@ class PlaceholderTypeBase: __instance = None __slots__ = () + def __init_subclass__(cls, *args, **kwargs): + raise TypeError(f"type '{cls.__name__}' is not an acceptable base type") + def __new__(cls): if cls.__instance is None: cls.__instance = object.__new__(cls) @@ -296,16 +299,9 @@ def __repr__(self): def __reduce__(self): return 'Placeholder' -def placeholder_init_subclass(cls, *args, **kwargs): - raise TypeError(f"type '{cls.__name__}' is not an acceptable base type") - -Placeholder = type( - 'PlaceholderType', - (PlaceholderTypeBase,), - {'__slots__': (), '__init_subclass__': placeholder_init_subclass} -)() -del PlaceholderTypeBase, placeholder_init_subclass +Placeholder = PlaceholderType() +del PlaceholderType def _partial_prepare_new(cls, func, args, keywords): if args: From 11855109c751a562b725b28ab5954d5321a3a6ad Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 25 Jun 2024 21:57:57 +0300 Subject: [PATCH 42/70] more optimal python functools.partial --- Doc/library/functools.rst | 4 +- Lib/functools.py | 169 ++++++++++++++++----------- Lib/inspect.py | 4 +- Lib/test/test_asyncio/test_events.py | 4 +- Modules/_functoolsmodule.c | 2 - 5 files changed, 105 insertions(+), 78 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index ee5db4bf1f10b0..0ba13270855b1c 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -330,15 +330,13 @@ The :mod:`functools` module defines the following functions: .. function:: partial(func, /, *args, **keywords) - Return a new :ref:`partial object ` which when called + Return a new :ref:`partial object` which when called will behave like *func* called with the positional arguments *args* and keyword arguments *keywords*. If more arguments are supplied to the call, they are appended to *args*. If additional keyword arguments are supplied, they extend and override *keywords*. Roughly equivalent to:: - Placeholder = object() - def partial(func, /, *args, **keywords): def newfunc(*fargs, **fkeywords): newkeywords = {**keywords, **fkeywords} diff --git a/Lib/functools.py b/Lib/functools.py index 40637954542fea..72771f747fab39 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -17,6 +17,7 @@ from abc import get_cache_token from collections import namedtuple # import types, weakref # Deferred to single_dispatch() +from operator import itemgetter from reprlib import recursive_repr from _thread import RLock @@ -299,71 +300,77 @@ def __repr__(self): def __reduce__(self): return 'Placeholder' - Placeholder = PlaceholderType() del PlaceholderType +def _partial_prepare_merger(args): + j = len(args) + order = list(range(j)) + for i, a in enumerate(args): + if a is Placeholder: + order[i] = j + j += 1 + return itemgetter(*order) + def _partial_prepare_new(cls, func, args, keywords): - if args: - if args[-1] is Placeholder: - raise TypeError("trailing Placeholders are not allowed") - phcount = args.count(Placeholder) - else: - phcount = 0 + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + tot_args = args + phcount = 0 + merger = None if isinstance(func, cls): pto_args = func.args - pto_phcount = func.placeholder_count - # merge args with args of `func` which is `partial` + pto_phcount = func._phcount if pto_phcount and args: - tot_args = list(pto_args) + # merge args with args of `func` which is `partial` nargs = len(args) - pos = j = 0 - end = nargs if nargs < pto_phcount else pto_phcount - while j < end: - pos = tot_args.index(Placeholder, pos) - tot_args[pos] = args[j] - pos += 1 - j += 1 - if pto_phcount < nargs: - tot_args.extend(args[pto_phcount:]) - phcount += pto_phcount - end - args = tuple(tot_args) + pto_merger = func._merger + if nargs >= pto_phcount: + phcount = args.count(Placeholder) + tot_args = pto_merger(pto_args + args[:pto_phcount]) + tot_args += args[pto_phcount:] + else: + phcount = (pto_phcount - nargs) + tot_args = pto_args + args + (Placeholder,) * phcount + tot_args = pto_merger(tot_args) + elif pto_phcount: + # and not args + phcount = pto_phcount + tot_args = pto_args + merger = func._merger + elif args: + # and not pto_phcount + phcount = args.count(Placeholder) + tot_args = pto_args + args else: - phcount += pto_phcount - args = func.args + args + # not pto_phcount and not args + phcount = 0 + tot_args = pto_args keywords = {**func.keywords, **keywords} func = func.func - return func, args, keywords, phcount - -def _partial_prepare_call(self, args, keywords): - pto_phcount = self.placeholder_count - pto_args = self.args - if pto_phcount: - n = len(args) - if n < pto_phcount: + elif args: + phcount = args.count(Placeholder) + if phcount and merger is None: + merger = _partial_prepare_merger(tot_args) + return func, tot_args, keywords, phcount, merger + +def _partial_prepare_call_args(self, args): + phcount = self._phcount + if phcount: + nargs = len(args) + if nargs < phcount: raise TypeError( "missing positional arguments " "in 'partial' call; expected " - f"at least {pto_phcount}, got {n}") - pto_args = list(pto_args) - pos = j = 0 - while j < pto_phcount: - pos = pto_args.index(Placeholder, pos) - pto_args[pos] = args[j] - pos += 1 - j += 1 - args = args[pto_phcount:] if n > pto_phcount else () - keywords = {**self.keywords, **keywords} - return pto_args, args, keywords - -def _partial_repr(self): - cls = type(self) - module = cls.__module__ - qualname = cls.__qualname__ - args = [repr(self.func)] - args.extend(map(repr, self.args)) - args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) - return f"{module}.{qualname}({', '.join(args)})" + f"at least {phcount}, got {len(args)}") + if nargs > phcount: + merged_args = self._merger(self.args + args[:phcount]) + return merged_args, args[phcount:] + else: + merged_args = self._merger(self.args + args) + return merged_args, () + else: + return self.args, args # Purely functional, no descriptor behaviour class partial: @@ -371,26 +378,36 @@ class partial: and keywords. """ - __slots__ = ("func", "args", "keywords", "placeholder_count", + __slots__ = ("func", "args", "keywords", "_phcount", "_merger", "__dict__", "__weakref__") def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") - func, args, kwds, phcount = _partial_prepare_new(cls, func, args, - keywords) + func, args, keywords, phcount, merger = _partial_prepare_new( + cls, func, args, keywords) self = super().__new__(cls) self.func = func self.args = args - self.keywords = kwds - self.placeholder_count = phcount + self.keywords = keywords + self._phcount = phcount + self._merger = merger return self def __call__(self, /, *args, **keywords): - pargs, args, kwds = _partial_prepare_call(self, args, keywords) - return self.func(*pargs, *args, **kwds) + pto_args, args = _partial_prepare_call_args(self, args) + keywords = {**self.keywords, **keywords} + return self.func(*pto_args, *args, **keywords) - __repr__ = recursive_repr()(_partial_repr) + @recursive_repr() + def __repr__(self): + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -407,12 +424,14 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") + phcount = 0 + merger = None if args: if args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") phcount = args.count(Placeholder) - else: - phcount = 0 + if phcount: + merger = _partial_prepare_merger(args) args = tuple(args) # just in case it's a subclass if kwds is None: @@ -426,7 +445,8 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds - self.placeholder_count = phcount + self._phcount = phcount + self._merger = merger try: from _functools import partial, Placeholder @@ -449,21 +469,32 @@ def __new__(cls, func, /, *args, **keywords): # flattening is mandatory in order to place cls/self before all # other arguments # it's also more efficient since only one function will be called - func, args, kwds, phcount = _partial_prepare_new(cls, func, args, - keywords) + func, args, keywords, phcount, merger = _partial_prepare_new( + cls, func, args, keywords) self = super().__new__(cls) self.func = func self.args = args - self.keywords = kwds - self.placeholder_count = phcount + self.keywords = keywords + self._phcount = phcount + self._merger = merger return self - __repr__ = _partial_repr + def __repr__(self): + args = ", ".join(map(repr, self.args)) + keywords = ", ".join("{}={!r}".format(k, v) + for k, v in self.keywords.items()) + format_string = "{module}.{cls}({func}, {args}, {keywords})" + return format_string.format(module=self.__class__.__module__, + cls=self.__class__.__qualname__, + func=self.func, + args=args, + keywords=keywords) def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): - pargs, args, kwds = _partial_prepare_call(self, args, keywords) - return self.func(cls_or_self, *pargs, *args, **kwds) + pto_args, args = _partial_prepare_call_args(self, args) + keywords = {**self.keywords, **keywords} + return self.func(cls_or_self, *pto_args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ _method.__partialmethod__ = self return _method diff --git a/Lib/inspect.py b/Lib/inspect.py index b037b2e3e09d5f..31e090fc29f24d 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2029,7 +2029,7 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): # If positional-only parameter is bound by partial, # it effectively disappears from the signature # However, if it is a Placeholder it is not removed - if arg_value != functools.Placeholder: + if arg_value is not functools.Placeholder: new_params.pop(param_name) continue @@ -2053,7 +2053,7 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): else: # was passed as a positional argument # But do not remove if it is a Placeholder - if arg_value != functools.Placeholder: + if arg_value is not functools.Placeholder: new_params.pop(param.name) continue diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 05406b620c511b..88c85a36b5d448 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2363,8 +2363,8 @@ def test_handle_repr(self): filename, lineno = test_utils.get_function_source(method) h = asyncio.Handle(cb, (), self.loop) - cb_regex = r'' - cb_regex = fr'functools.partialmethod\({cb_regex}\)\(\)' + cb_regex = r'' + cb_regex = fr'functools.partialmethod\({cb_regex}, , \)\(\)' regex = fr'^$' self.assertRegex(repr(h), regex) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index ce3fde52cff383..58ba8b214ad4d4 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -553,8 +553,6 @@ static PyMemberDef partial_memberlist[] = { "tuple of arguments to future partial calls"}, {"keywords", _Py_T_OBJECT, OFF(kw), Py_READONLY, "dictionary of keyword arguments to future partial calls"}, - {"placeholder_count", Py_T_PYSSIZET, OFF(phcount), Py_READONLY, - "number of placeholders"}, {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(partialobject, weakreflist), Py_READONLY}, {"__dictoffset__", Py_T_PYSSIZET, From 266b4fa7d00540a9455c31f638c9e336cafb4605 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 26 Jun 2024 00:58:38 +0300 Subject: [PATCH 43/70] placeholder arg and pre-placeholder instance conversions to positional --- Doc/library/functools.rst | 2 -- Lib/inspect.py | 15 ++++++++++++--- Lib/test/test_inspect/test_inspect.py | 19 ++++++++++++++----- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 0ba13270855b1c..19a6e90ac009c2 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -366,8 +366,6 @@ The :mod:`functools` module defines the following functions: >>> from functools import partial, Placeholder >>> say_to_world = partial(print, Placeholder, Placeholder, "world!") - >>> say_to_world.placeholder_count - 2 >>> say_to_world('Hello', 'dear') Hello dear world! diff --git a/Lib/inspect.py b/Lib/inspect.py index 31e090fc29f24d..cf82f2d599efb0 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2052,9 +2052,13 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): new_params[param_name] = param.replace(default=arg_value) else: # was passed as a positional argument - # But do not remove if it is a Placeholder - if arg_value is not functools.Placeholder: - new_params.pop(param.name) + # Do not pop if it is a Placeholder + # and change kind to positional only + if arg_value is functools.Placeholder: + new_param = param.replace(kind=_POSITIONAL_ONLY) + new_params[param_name] = new_param + else: + new_params.pop(param_name) continue if param.kind is _KEYWORD_ONLY: @@ -2548,6 +2552,11 @@ def _signature_from_callable(obj, *, sig_params = tuple(sig.parameters.values()) assert (not sig_params or first_wrapped_param is not sig_params[0]) + # If there were placeholders set, + # first param is transformaed to positional only + if partialmethod.args.count(functools.Placeholder): + first_wrapped_param = first_wrapped_param.replace( + kind=Parameter.POSITIONAL_ONLY) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 26455bab396ac7..503d9e543beb45 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3311,13 +3311,13 @@ def test(a, b, *, c, d): # With Placeholder self.assertEqual(self.signature(partial(test, Placeholder, 1)), - ((('a', ..., ..., "positional_or_keyword"), + ((('a', ..., ..., "positional_only"), ('c', ..., ..., "keyword_only"), ('d', ..., ..., "keyword_only")), ...)) self.assertEqual(self.signature(partial(test, Placeholder, 1, c=2)), - ((('a', ..., ..., "positional_or_keyword"), + ((('a', ..., ..., "positional_only"), ('c', 2, ..., "keyword_only"), ('d', ..., ..., "keyword_only")), ...)) @@ -3369,6 +3369,15 @@ def test(a, *args, b, **kwargs): ('kwargs', ..., ..., "var_keyword")), ...)) + # With Placeholder + p = partial(test, Placeholder, Placeholder, 1, b=0, test=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + def test(a, b, c:int) -> 42: pass @@ -3505,12 +3514,12 @@ def test(it, a, b, *, c) -> 'spam': # With Placeholder self.assertEqual(self.signature(Spam.bar, eval_str=False), - ((('it', ..., ..., 'positional_or_keyword'), - ('a', ..., ..., 'positional_or_keyword'), + ((('it', ..., ..., 'positional_only'), + ('a', ..., ..., 'positional_only'), ('c', 1, ..., 'keyword_only')), 'spam')) self.assertEqual(self.signature(Spam().bar, eval_str=False), - ((('a', ..., ..., 'positional_or_keyword'), + ((('a', ..., ..., 'positional_only'), ('c', 1, ..., 'keyword_only')), 'spam')) From dd58a12e59c8780606c63289b197bdcc502e8259 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 26 Jun 2024 01:05:04 +0300 Subject: [PATCH 44/70] unittest.mock.ANY test for Placeholder --- Lib/test/test_inspect/test_inspect.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 503d9e543beb45..0077c891eb5ca8 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3322,6 +3322,12 @@ def test(a, b, *, c, d): ('d', ..., ..., "keyword_only")), ...)) + # Ensure unittest.mock.ANY & similar do not get picked up as a Placeholder + self.assertEqual(self.signature(partial(test, unittest.mock.ANY, 1, c=2)), + ((('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + def test(a, *args, b, **kwargs): pass From 5971fbb40c6833f136634fcfb5d827bf652bf138 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 26 Jun 2024 06:09:38 +0300 Subject: [PATCH 45/70] functools.partial.__get__ --- Lib/functools.py | 6 ++++++ Lib/inspect.py | 11 +++++++---- Lib/test/test_inspect/test_inspect.py | 6 +++--- Modules/_functoolsmodule.c | 9 +++++++++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 72771f747fab39..a3285291fb25a4 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -19,6 +19,7 @@ # import types, weakref # Deferred to single_dispatch() from operator import itemgetter from reprlib import recursive_repr +from types import MethodType from _thread import RLock # Avoid importing types, so we can speedup import time @@ -381,6 +382,11 @@ class partial: __slots__ = ("func", "args", "keywords", "_phcount", "_merger", "__dict__", "__weakref__") + def __get__(self, obj, objtype=None): + if obj is None: + return self + return MethodType(self, obj) + def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") diff --git a/Lib/inspect.py b/Lib/inspect.py index cf82f2d599efb0..7bcb089fc44220 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2567,14 +2567,17 @@ def _signature_from_callable(obj, *, skip_bound_arg=skip_bound_arg, globals=globals, locals=locals, eval_str=eval_str) - if _signature_is_builtin(obj): - return _signature_from_builtin(sigcls, obj, - skip_bound_arg=skip_bound_arg) - if isinstance(obj, functools.partial): + # Must be before _signature_is_builtin + # as it uses `ismethoddescriptor` + # while partial has __get__ implemented wrapped_sig = _get_signature_of(obj.func) return _signature_get_partial(wrapped_sig, obj) + if _signature_is_builtin(obj): + return _signature_from_builtin(sigcls, obj, + skip_bound_arg=skip_bound_arg) + if isinstance(obj, type): # obj is a class or a metaclass diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 0077c891eb5ca8..0a8af7304d7b11 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3681,7 +3681,7 @@ def __init__(self, b): with self.subTest('partial'): class CM(type): - __call__ = functools.partial(lambda x, a: (x, a), 2) + __call__ = staticmethod(functools.partial(lambda x, a: (x, a), 2)) class C(metaclass=CM): def __init__(self, b): pass @@ -3835,7 +3835,7 @@ class C: with self.subTest('partial'): class C: - __init__ = functools.partial(lambda x, a: None, 2) + __init__ = staticmethod(functools.partial(lambda x, a: None, 2)) C(1) # does not raise self.assertEqual(self.signature(C), @@ -4093,7 +4093,7 @@ class C: with self.subTest('partial'): class C: - __call__ = functools.partial(lambda x, a: (x, a), 2) + __call__ = staticmethod(functools.partial(lambda x, a: (x, a), 2)) self.assertEqual(C()(1), (2, 1)) self.assertEqual(self.signature(C()), diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 58ba8b214ad4d4..7f23b810b3b3ac 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -331,6 +331,14 @@ partial_dealloc(partialobject *pto) Py_DECREF(tp); } +static PyObject * +partial_descr_get(PyObject *pto, PyObject *obj, PyObject *type) +{ + if (obj == Py_None || obj == NULL) { + return Py_NewRef(pto); + } + return PyMethod_New(pto, obj); +} /* Merging keyword arguments using the vectorcall convention is messy, so * if we would need to do that, we stop using vectorcall and fall back @@ -736,6 +744,7 @@ static PyType_Slot partial_type_slots[] = { {Py_tp_methods, partial_methods}, {Py_tp_members, partial_memberlist}, {Py_tp_getset, partial_getsetlist}, + {Py_tp_descr_get, (descrgetfunc)partial_descr_get}, {Py_tp_new, partial_new}, {Py_tp_free, PyObject_GC_Del}, {0, 0} From 9033650d8a625efa6f48e54c0d0226455a531232 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 26 Jun 2024 21:32:31 +0300 Subject: [PATCH 46/70] Revert "functools.partial.__get__" This reverts commit 5971fbb40c6833f136634fcfb5d827bf652bf138. --- Lib/functools.py | 6 ------ Lib/inspect.py | 11 ++++------- Lib/test/test_inspect/test_inspect.py | 6 +++--- Modules/_functoolsmodule.c | 9 --------- 4 files changed, 7 insertions(+), 25 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index a3285291fb25a4..72771f747fab39 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -19,7 +19,6 @@ # import types, weakref # Deferred to single_dispatch() from operator import itemgetter from reprlib import recursive_repr -from types import MethodType from _thread import RLock # Avoid importing types, so we can speedup import time @@ -382,11 +381,6 @@ class partial: __slots__ = ("func", "args", "keywords", "_phcount", "_merger", "__dict__", "__weakref__") - def __get__(self, obj, objtype=None): - if obj is None: - return self - return MethodType(self, obj) - def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") diff --git a/Lib/inspect.py b/Lib/inspect.py index 7bcb089fc44220..cf82f2d599efb0 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2567,17 +2567,14 @@ def _signature_from_callable(obj, *, skip_bound_arg=skip_bound_arg, globals=globals, locals=locals, eval_str=eval_str) - if isinstance(obj, functools.partial): - # Must be before _signature_is_builtin - # as it uses `ismethoddescriptor` - # while partial has __get__ implemented - wrapped_sig = _get_signature_of(obj.func) - return _signature_get_partial(wrapped_sig, obj) - if _signature_is_builtin(obj): return _signature_from_builtin(sigcls, obj, skip_bound_arg=skip_bound_arg) + if isinstance(obj, functools.partial): + wrapped_sig = _get_signature_of(obj.func) + return _signature_get_partial(wrapped_sig, obj) + if isinstance(obj, type): # obj is a class or a metaclass diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 0a8af7304d7b11..0077c891eb5ca8 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3681,7 +3681,7 @@ def __init__(self, b): with self.subTest('partial'): class CM(type): - __call__ = staticmethod(functools.partial(lambda x, a: (x, a), 2)) + __call__ = functools.partial(lambda x, a: (x, a), 2) class C(metaclass=CM): def __init__(self, b): pass @@ -3835,7 +3835,7 @@ class C: with self.subTest('partial'): class C: - __init__ = staticmethod(functools.partial(lambda x, a: None, 2)) + __init__ = functools.partial(lambda x, a: None, 2) C(1) # does not raise self.assertEqual(self.signature(C), @@ -4093,7 +4093,7 @@ class C: with self.subTest('partial'): class C: - __call__ = staticmethod(functools.partial(lambda x, a: (x, a), 2)) + __call__ = functools.partial(lambda x, a: (x, a), 2) self.assertEqual(C()(1), (2, 1)) self.assertEqual(self.signature(C()), diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 7f23b810b3b3ac..58ba8b214ad4d4 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -331,14 +331,6 @@ partial_dealloc(partialobject *pto) Py_DECREF(tp); } -static PyObject * -partial_descr_get(PyObject *pto, PyObject *obj, PyObject *type) -{ - if (obj == Py_None || obj == NULL) { - return Py_NewRef(pto); - } - return PyMethod_New(pto, obj); -} /* Merging keyword arguments using the vectorcall convention is messy, so * if we would need to do that, we stop using vectorcall and fall back @@ -744,7 +736,6 @@ static PyType_Slot partial_type_slots[] = { {Py_tp_methods, partial_methods}, {Py_tp_members, partial_memberlist}, {Py_tp_getset, partial_getsetlist}, - {Py_tp_descr_get, (descrgetfunc)partial_descr_get}, {Py_tp_new, partial_new}, {Py_tp_free, PyObject_GC_Del}, {0, 0} From a3d39b033bd467502e3b18800bb58af6e2c614aa Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 26 Jun 2024 23:12:27 +0300 Subject: [PATCH 47/70] review changes --- Doc/library/functools.rst | 7 ++--- Lib/functools.py | 54 ++++++++++++++++++-------------------- Lib/pickle.py | 4 +-- Modules/_functoolsmodule.c | 12 +-------- Modules/_pickle.c | 9 ------- 5 files changed, 30 insertions(+), 56 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 19a6e90ac009c2..91eb2aee9cda76 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -391,11 +391,8 @@ The :mod:`functools` module defines the following functions: .. data:: Placeholder A singleton object used as a sentinel to represent a - "gap" in positional arguments when calling :func:`partial`. - This object has features similar to ``None``: - - >>> type(Placeholder)() is Placeholder - True + "gap" in positional arguments when calling :func:`partial` + and :func:`partialmethod`. .. class:: partialmethod(func, /, *args, **keywords) diff --git a/Lib/functools.py b/Lib/functools.py index b92aaba9b93c8b..0cf4eb6a6908c1 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -275,7 +275,7 @@ def reduce(function, sequence, initial=_initial_missing): ################################################################################ -class PlaceholderType: +class _PlaceholderType: """The type of the Placeholder singleton. Used as a placeholder for partial arguments. @@ -291,17 +291,13 @@ def __new__(cls): cls.__instance = object.__new__(cls) return cls.__instance - def __bool__(self): - raise TypeError("Placeholder should not be used in a boolean context") - def __repr__(self): return 'Placeholder' def __reduce__(self): return 'Placeholder' -Placeholder = PlaceholderType() -del PlaceholderType +Placeholder = _PlaceholderType() def _partial_prepare_merger(args): j = len(args) @@ -330,7 +326,7 @@ def _partial_prepare_new(cls, func, args, keywords): tot_args = pto_merger(pto_args + args[:pto_phcount]) tot_args += args[pto_phcount:] else: - phcount = (pto_phcount - nargs) + phcount = pto_phcount - nargs tot_args = pto_args + args + (Placeholder,) * phcount tot_args = pto_merger(tot_args) elif pto_phcount: @@ -354,24 +350,6 @@ def _partial_prepare_new(cls, func, args, keywords): merger = _partial_prepare_merger(tot_args) return func, tot_args, keywords, phcount, merger -def _partial_prepare_call_args(self, args): - phcount = self._phcount - if phcount: - nargs = len(args) - if nargs < phcount: - raise TypeError( - "missing positional arguments " - "in 'partial' call; expected " - f"at least {phcount}, got {len(args)}") - if nargs > phcount: - merged_args = self._merger(self.args + args[:phcount]) - return merged_args, args[phcount:] - else: - merged_args = self._merger(self.args + args) - return merged_args, () - else: - return self.args, args - # Purely functional, no descriptor behaviour class partial: """New function with partial application of the given arguments @@ -395,7 +373,17 @@ def __new__(cls, func, /, *args, **keywords): return self def __call__(self, /, *args, **keywords): - pto_args, args = _partial_prepare_call_args(self, args) + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args[:phcount]) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partial' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} return self.func(*pto_args, *args, **keywords) @@ -449,7 +437,7 @@ def __setstate__(self, state): self._merger = merger try: - from _functools import partial, Placeholder + from _functools import partial, Placeholder, _PlaceholderType except ImportError: pass @@ -490,7 +478,17 @@ def __repr__(self): def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): - pto_args, args = _partial_prepare_call_args(self, args) + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args[:phcount]) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partial' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} return self.func(cls_or_self, *pto_args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ diff --git a/Lib/pickle.py b/Lib/pickle.py index f317247f167ad3..d719ceb7a0b8e8 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -27,7 +27,7 @@ from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache from itertools import islice -from functools import partial, Placeholder +from functools import partial import sys from sys import maxsize from struct import pack, unpack @@ -1140,8 +1140,6 @@ def save_type(self, obj): return self.save_reduce(type, (NotImplemented,), obj=obj) elif obj is type(...): return self.save_reduce(type, (...,), obj=obj) - elif obj is type(Placeholder): - return self.save_reduce(type, (Placeholder,), obj=obj) return self.save_global(obj) dispatch[FunctionType] = save_global diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 58ba8b214ad4d4..bab414ede6d656 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -97,27 +97,17 @@ placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return placeholder_instance; } -static int -placeholder_bool(PyObject *v) -{ - PyErr_SetString(PyExc_TypeError, - "Placeholder should not be used in a boolean context"); - return -1; -} - static PyType_Slot placeholder_type_slots[] = { {Py_tp_dealloc, placeholder_dealloc}, {Py_tp_repr, placeholder_repr}, {Py_tp_doc, (void *)placeholder_doc}, {Py_tp_methods, placeholder_methods}, {Py_tp_new, placeholder_new}, - // Number protocol - {Py_nb_bool, placeholder_bool}, {0, 0} }; static PyType_Spec placeholder_type_spec = { - .name = "functools.PlaceholderType", + .name = "functools._PlaceholderType", .basicsize = sizeof(placeholderobject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, .slots = placeholder_type_slots diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 01072ea77ed7e6..21be88a79d8705 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -198,7 +198,6 @@ typedef struct { /* functools.partial, used for implementing __newobj_ex__ with protocols 2 and 3 */ PyObject *partial; - PyObject *placeholder; /* Types */ PyTypeObject *Pickler_Type; @@ -254,7 +253,6 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->codecs_encode); Py_CLEAR(st->getattr); Py_CLEAR(st->partial); - Py_CLEAR(st->placeholder); Py_CLEAR(st->Pickler_Type); Py_CLEAR(st->Unpickler_Type); Py_CLEAR(st->Pdata_Type); @@ -377,9 +375,6 @@ _Pickle_InitState(PickleState *st) st->partial = _PyImport_GetModuleAttrString("functools", "partial"); if (!st->partial) goto error; - st->placeholder = _PyImport_GetModuleAttrString("functools", "Placeholder"); - if (!st->placeholder) - goto error; return 0; @@ -3865,9 +3860,6 @@ save_type(PickleState *state, PicklerObject *self, PyObject *obj) else if (obj == (PyObject *)&_PyNotImplemented_Type) { return save_singleton_type(state, self, obj, Py_NotImplemented); } - else if (obj == (PyObject *)Py_TYPE(state->placeholder)) { - return save_singleton_type(state, self, obj, state->placeholder); - } return save_global(state, self, obj, NULL); } @@ -7800,7 +7792,6 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->codecs_encode); Py_VISIT(st->getattr); Py_VISIT(st->partial); - Py_VISIT(st->placeholder); Py_VISIT(st->Pickler_Type); Py_VISIT(st->Unpickler_Type); Py_VISIT(st->Pdata_Type); From 9e4c5df60155f8e271f4249c5036ef8e9a9905dc Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 27 Jun 2024 11:25:53 +0300 Subject: [PATCH 48/70] factor out repr. same to be used for partial and partialmethod --- Lib/functools.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 0cf4eb6a6908c1..31c00693d204e3 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -350,6 +350,15 @@ def _partial_prepare_new(cls, func, args, keywords): merger = _partial_prepare_merger(tot_args) return func, tot_args, keywords, phcount, merger +def _partial_repr(self): + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" + # Purely functional, no descriptor behaviour class partial: """New function with partial application of the given arguments @@ -387,15 +396,7 @@ def __call__(self, /, *args, **keywords): keywords = {**self.keywords, **keywords} return self.func(*pto_args, *args, **keywords) - @recursive_repr() - def __repr__(self): - cls = type(self) - module = cls.__module__ - qualname = cls.__qualname__ - args = [repr(self.func)] - args.extend(map(repr, self.args)) - args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) - return f"{module}.{qualname}({', '.join(args)})" + __repr__ = recursive_repr()(_partial_repr) def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -467,14 +468,7 @@ def __new__(cls, func, /, *args, **keywords): self._merger = merger return self - def __repr__(self): - cls = type(self) - module = cls.__module__ - qualname = cls.__qualname__ - args = [repr(self.func)] - args.extend(map(repr, self.args)) - args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) - return f"{module}.{qualname}({', '.join(args)})" + __repr__ = _partial_repr def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): From 16f12f8d9b5640ad7928f25e35a9337c69ae619d Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 27 Jun 2024 13:28:27 +0300 Subject: [PATCH 49/70] microopt & args.count(Placeholder) can not be used as it uses __eq__ --- Lib/functools.py | 66 ++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 31c00693d204e3..34ec7e34a83fa9 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -300,54 +300,56 @@ def __reduce__(self): Placeholder = _PlaceholderType() def _partial_prepare_merger(args): - j = len(args) - order = list(range(j)) - for i, a in enumerate(args): + order = list() + nargs = j = len(args) + i = 0 + for a in args: if a is Placeholder: - order[i] = j + order.append(j) j += 1 - return itemgetter(*order) + else: + order.append(i) + i += 1 + phcount = j - nargs + merger = None + if phcount: + merger = itemgetter(*order) + return phcount, merger def _partial_prepare_new(cls, func, args, keywords): if args and args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") - tot_args = args - phcount = 0 - merger = None if isinstance(func, cls): - pto_args = func.args pto_phcount = func._phcount if pto_phcount and args: # merge args with args of `func` which is `partial` nargs = len(args) - pto_merger = func._merger - if nargs >= pto_phcount: - phcount = args.count(Placeholder) - tot_args = pto_merger(pto_args + args[:pto_phcount]) + tot_args = func.args + args + if nargs < pto_phcount: + tot_args += (Placeholder,) * (pto_phcount - nargs) + tot_args = func._merger(tot_args) + if nargs > pto_phcount: tot_args += args[pto_phcount:] - else: - phcount = pto_phcount - nargs - tot_args = pto_args + args + (Placeholder,) * phcount - tot_args = pto_merger(tot_args) + phcount, merger = _partial_prepare_merger(tot_args) elif pto_phcount: # and not args + tot_args = func.args phcount = pto_phcount - tot_args = pto_args merger = func._merger elif args: # and not pto_phcount - phcount = args.count(Placeholder) - tot_args = pto_args + args + tot_args = func.args + args + phcount, merger = _partial_prepare_merger(tot_args) else: # not pto_phcount and not args - phcount = 0 - tot_args = pto_args + tot_args = func.args + phcount, merger = 0, None keywords = {**func.keywords, **keywords} func = func.func - elif args: - phcount = args.count(Placeholder) - if phcount and merger is None: - merger = _partial_prepare_merger(tot_args) + elif tot_args := args: + phcount, merger = _partial_prepare_merger(tot_args) + else: + phcount, merger = 0, None return func, tot_args, keywords, phcount, merger def _partial_repr(self): @@ -385,7 +387,7 @@ def __call__(self, /, *args, **keywords): phcount = self._phcount if phcount: try: - pto_args = self._merger(self.args + args[:phcount]) + pto_args = self._merger(self.args + args) args = args[phcount:] except IndexError: raise TypeError("missing positional arguments " @@ -413,14 +415,12 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") - phcount = 0 - merger = None if args: if args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") - phcount = args.count(Placeholder) - if phcount: - merger = _partial_prepare_merger(args) + phcount, merger = _partial_prepare_merger(args) + else: + phcount, merger = 0, None args = tuple(args) # just in case it's a subclass if kwds is None: @@ -475,7 +475,7 @@ def _method(cls_or_self, /, *args, **keywords): phcount = self._phcount if phcount: try: - pto_args = self._merger(self.args + args[:phcount]) + pto_args = self._merger(self.args + args) args = args[phcount:] except IndexError: raise TypeError("missing positional arguments " From 82dd600c85be5ff6c14ea11b4e5755c811389caf Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 27 Jun 2024 13:43:31 +0300 Subject: [PATCH 50/70] simplify preparation --- Lib/functools.py | 59 ++++++++++++++++++++---------------------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 34ec7e34a83fa9..17821e8c5a940d 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -300,9 +300,11 @@ def __reduce__(self): Placeholder = _PlaceholderType() def _partial_prepare_merger(args): + nargs = len(args) + if not nargs: + return 0, None order = list() - nargs = j = len(args) - i = 0 + i, j = 0, nargs for a in args: if a is Placeholder: order.append(j) @@ -311,9 +313,7 @@ def _partial_prepare_merger(args): order.append(i) i += 1 phcount = j - nargs - merger = None - if phcount: - merger = itemgetter(*order) + merger = itemgetter(*order) if phcount else None return phcount, merger def _partial_prepare_new(cls, func, args, keywords): @@ -321,35 +321,27 @@ def _partial_prepare_new(cls, func, args, keywords): raise TypeError("trailing Placeholders are not allowed") if isinstance(func, cls): pto_phcount = func._phcount - if pto_phcount and args: - # merge args with args of `func` which is `partial` - nargs = len(args) - tot_args = func.args + args - if nargs < pto_phcount: - tot_args += (Placeholder,) * (pto_phcount - nargs) - tot_args = func._merger(tot_args) - if nargs > pto_phcount: - tot_args += args[pto_phcount:] - phcount, merger = _partial_prepare_merger(tot_args) - elif pto_phcount: - # and not args - tot_args = func.args - phcount = pto_phcount - merger = func._merger - elif args: - # and not pto_phcount - tot_args = func.args + args + tot_args = func.args + if args: + tot_args += args + if pto_phcount: + # merge args with args of `func` which is `partial` + nargs = len(args) + if nargs < pto_phcount: + tot_args += (Placeholder,) * (pto_phcount - nargs) + tot_args = func._merger(tot_args) + if nargs > pto_phcount: + tot_args += args[pto_phcount:] phcount, merger = _partial_prepare_merger(tot_args) - else: - # not pto_phcount and not args - tot_args = func.args + elif pto_phcount: # not args + phcount, merger = pto_phcount, func._merger + else: # not args and not pto_phcount phcount, merger = 0, None keywords = {**func.keywords, **keywords} func = func.func - elif tot_args := args: - phcount, merger = _partial_prepare_merger(tot_args) else: - phcount, merger = 0, None + tot_args = args + phcount, merger = _partial_prepare_merger(tot_args) return func, tot_args, keywords, phcount, merger def _partial_repr(self): @@ -415,12 +407,9 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") - if args: - if args[-1] is Placeholder: - raise TypeError("trailing Placeholders are not allowed") - phcount, merger = _partial_prepare_merger(args) - else: - phcount, merger = 0, None + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + phcount, merger = _partial_prepare_merger(args) args = tuple(args) # just in case it's a subclass if kwds is None: From f9cb65388361b76697711179d29120ae08485372 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 27 Jun 2024 16:40:03 +0300 Subject: [PATCH 51/70] whatsnew + minor doc edit --- Doc/library/functools.rst | 4 ++-- Doc/whatsnew/3.14.rst | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 91eb2aee9cda76..7088130cdb6763 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -390,8 +390,8 @@ The :mod:`functools` module defines the following functions: .. data:: Placeholder - A singleton object used as a sentinel to represent a - "gap" in positional arguments when calling :func:`partial` + A singleton object used as a sentinel to reserve a place + for positional arguments when calling :func:`partial` and :func:`partialmethod`. .. class:: partialmethod(func, /, *args, **keywords) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9662044915b8ca..c820792d06a753 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -92,6 +92,12 @@ ast Added :func:`ast.compare` for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) +functools +--------- +* :func:`functools.partial` and :functools:`partialmethod` now support + :data:`functools.Placeholder` sentinels to reserve a place for + positional arguments. + os -- From d255524a2e0b6fd6a4b3334415dc3ddb64c8377b Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 27 Jun 2024 16:43:57 +0300 Subject: [PATCH 52/70] typo fixes --- Doc/whatsnew/3.14.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index c820792d06a753..5647389c09a930 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -94,7 +94,8 @@ Added :func:`ast.compare` for comparing two ASTs. functools --------- -* :func:`functools.partial` and :functools:`partialmethod` now support + +* :func:`functools.partial` and :func:`functools.partialmethod` now support :data:`functools.Placeholder` sentinels to reserve a place for positional arguments. From 404044e6c839b9e0b4c7aca3881153cb6a718c47 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Thu, 27 Jun 2024 17:49:35 +0300 Subject: [PATCH 53/70] whatsnew edit --- Doc/whatsnew/3.14.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5647389c09a930..0840751ea7411c 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -98,6 +98,7 @@ functools * :func:`functools.partial` and :func:`functools.partialmethod` now support :data:`functools.Placeholder` sentinels to reserve a place for positional arguments. + (Contributed by Dominykas Grigonis in :gh:`119127`) os -- From 800217b43818615e9898db8dff73194203f32d35 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Fri, 28 Jun 2024 02:47:58 +0300 Subject: [PATCH 54/70] revert stylistic changes --- Lib/functools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 17821e8c5a940d..71e57d2f31222a 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -403,8 +403,8 @@ def __setstate__(self, state): raise TypeError(f"expected 4 items in state, got {len(state)}") func, args, kwds, namespace = state if (not callable(func) or not isinstance(args, tuple) or - (kwds is not None and not isinstance(kwds, dict)) or - (namespace is not None and not isinstance(namespace, dict))): + (kwds is not None and not isinstance(kwds, dict)) or + (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") if args and args[-1] is Placeholder: From 38ee450f242ae998e6b50db6456807d9125a5a28 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Fri, 28 Jun 2024 16:27:51 +0300 Subject: [PATCH 55/70] factor out full __new__ --- Lib/functools.py | 64 ++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 71e57d2f31222a..f6e8dbb30633c7 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -316,7 +316,19 @@ def _partial_prepare_merger(args): merger = itemgetter(*order) if phcount else None return phcount, merger -def _partial_prepare_new(cls, func, args, keywords): +def _partial_new(cls, func, /, *args, **keywords): + if issubclass(cls, partial): + if not callable(func): + raise TypeError("the first argument must be callable") + else: + # assert issubclass(cls, partialmethod) + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError(f"{func!r} is not callable or a descriptor") + # func could be a descriptor like classmethod which isn't callable, + # so we can't inherit from partial (it verifies func is callable) + # flattening is mandatory in order to place cls/self before all + # other arguments + # it's also more efficient since only one function will be called if args and args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") if isinstance(func, cls): @@ -342,7 +354,14 @@ def _partial_prepare_new(cls, func, args, keywords): else: tot_args = args phcount, merger = _partial_prepare_merger(tot_args) - return func, tot_args, keywords, phcount, merger + + self = object.__new__(cls) + self.func = func + self.args = tot_args + self.keywords = keywords + self._phcount = phcount + self._merger = merger + return self def _partial_repr(self): cls = type(self) @@ -362,22 +381,11 @@ class partial: __slots__ = ("func", "args", "keywords", "_phcount", "_merger", "__dict__", "__weakref__") - def __new__(cls, func, /, *args, **keywords): - if not callable(func): - raise TypeError("the first argument must be callable") - func, args, keywords, phcount, merger = _partial_prepare_new( - cls, func, args, keywords) - self = super().__new__(cls) - self.func = func - self.args = args - self.keywords = keywords - self._phcount = phcount - self._merger = merger - return self + __new__ = _partial_new + __repr__ = recursive_repr()(_partial_repr) def __call__(self, /, *args, **keywords): - phcount = self._phcount - if phcount: + if phcount := self._phcount: try: pto_args = self._merger(self.args + args) args = args[phcount:] @@ -390,8 +398,6 @@ def __call__(self, /, *args, **keywords): keywords = {**self.keywords, **keywords} return self.func(*pto_args, *args, **keywords) - __repr__ = recursive_repr()(_partial_repr) - def __reduce__(self): return type(self), (self.func,), (self.func, self.args, self.keywords or None, self.__dict__ or None) @@ -439,30 +445,12 @@ class partialmethod: Supports wrapping existing descriptors and handles non-descriptor callables as instance methods. """ - def __new__(cls, func, /, *args, **keywords): - if not callable(func) and not hasattr(func, "__get__"): - raise TypeError(f"{func!r} is not callable or a descriptor") - # func could be a descriptor like classmethod which isn't callable, - # so we can't inherit from partial (it verifies func is callable) - # flattening is mandatory in order to place cls/self before all - # other arguments - # it's also more efficient since only one function will be called - func, args, keywords, phcount, merger = _partial_prepare_new( - cls, func, args, keywords) - self = super().__new__(cls) - self.func = func - self.args = args - self.keywords = keywords - self._phcount = phcount - self._merger = merger - return self - + __new__ = _partial_new __repr__ = _partial_repr def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): - phcount = self._phcount - if phcount: + if phcount := self._phcount: try: pto_args = self._merger(self.args + args) args = args[phcount:] From fd16189b7f8262905e37bd4a636f7bcf143aada4 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 12 Aug 2024 06:37:57 +0300 Subject: [PATCH 56/70] CR part 1 --- Lib/functools.py | 20 +++++++++---------- Lib/test/test_functools.py | 28 +++++++++++++++++++++++++++ Lib/test/test_inspect/test_inspect.py | 28 +++++++++++++++++++++++++++ Modules/_functoolsmodule.c | 24 +++++++++++------------ 4 files changed, 78 insertions(+), 22 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index fa66f2d2296eb9..4181d7ed55b901 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -304,7 +304,7 @@ def _partial_prepare_merger(args): nargs = len(args) if not nargs: return 0, None - order = list() + order = [] i, j = 0, nargs for a in args: if a is Placeholder: @@ -319,20 +319,18 @@ def _partial_prepare_merger(args): def _partial_new(cls, func, /, *args, **keywords): if issubclass(cls, partial): + base_cls = partial if not callable(func): raise TypeError("the first argument must be callable") else: + base_cls = partialmethod + # func could be a descriptor like classmethod which isn't callable # assert issubclass(cls, partialmethod) if not callable(func) and not hasattr(func, "__get__"): raise TypeError(f"{func!r} is not callable or a descriptor") - # func could be a descriptor like classmethod which isn't callable, - # so we can't inherit from partial (it verifies func is callable) - # flattening is mandatory in order to place cls/self before all - # other arguments - # it's also more efficient since only one function will be called if args and args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") - if isinstance(func, cls): + if isinstance(func, base_cls): pto_phcount = func._phcount tot_args = func.args if args: @@ -346,7 +344,7 @@ def _partial_new(cls, func, /, *args, **keywords): if nargs > pto_phcount: tot_args += args[pto_phcount:] phcount, merger = _partial_prepare_merger(tot_args) - elif pto_phcount: # not args + elif pto_phcount: # not args and pto_phcount phcount, merger = pto_phcount, func._merger else: # not args and not pto_phcount phcount, merger = 0, None @@ -386,7 +384,8 @@ class partial: __repr__ = recursive_repr()(_partial_repr) def __call__(self, /, *args, **keywords): - if phcount := self._phcount: + phcount = self._phcount + if phcount: try: pto_args = self._merger(self.args + args) args = args[phcount:] @@ -456,7 +455,8 @@ class partialmethod: def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): - if phcount := self._phcount: + phcount = self._phcount + if phcount: try: pto_args = self._merger(self.args + args) args = args[phcount:] diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 3f3ceb7f7ff6c6..a86a7f6fbe1c31 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -510,6 +510,19 @@ def __str__(self): self.assertIn('astr', r) self.assertIn("['sth']", r) + def test_placeholders_refcount_smoke(self): + PH = self.module.Placeholder + # sum supports vector call + lst1, start = [], [] + sum_lists = self.partial(sum, PH, start) + for i in range(10): + sum_lists([lst1, lst1]) + # collections.ChainMap initializer does not support vectorcall + map1, map2 = {}, {} + partial_cm = self.partial(collections.ChainMap, PH, map1) + for i in range(10): + partial_cm(map2, map2) + class TestPartialPy(TestPartial, unittest.TestCase): module = py_functools @@ -534,6 +547,13 @@ class TestPartialCSubclass(TestPartialC): class TestPartialPySubclass(TestPartialPy): partial = PyPartialSubclass + def test_subclass_optimization(self): + p = py_functools.partial(min, 2) + p2 = self.partial(p, 1) + assert p2.func == min + assert p2(0) == 0 + + class TestPartialMethod(unittest.TestCase): class A(object): @@ -671,6 +691,14 @@ def f(a, b, /): p = functools.partial(f, 1) self.assertEqual(p(2), f(1, 2)) + def test_subclass_optimization(self): + class PartialMethodSubclass(functools.partialmethod): + pass + p = functools.partialmethod(min, 2) + p2 = PartialMethodSubclass(p, 1) + assert p2.func == min + assert p2.__get__(0)() == 0 + class TestUpdateWrapper(unittest.TestCase): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 47b6fd2b357fd7..20c5a1d98590ab 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3507,6 +3507,34 @@ def foo(a, b, /, c, d, **kwargs): ('kwargs', ..., ..., 'var_keyword')), ...)) + # Positional only With Placeholder + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + # Optionals Positional With Placeholder + def foo(a=0, b=1, /, c=2, d=3): + pass + + # Positional + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', 0, ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only")), + ...)) + + # Positional or Keyword - transformed to positional + p = partial(foo, Placeholder, 1, Placeholder, 1) + self.assertEqual(self.signature(p), + ((('a', 0, ..., "positional_only"), + ('c', 2, ..., "positional_only")), + ...)) + def test_signature_on_partialmethod(self): from functools import partialmethod diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 09e23f1360469b..251c4dbc86fc8c 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -194,7 +194,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) pto->fn = Py_NewRef(func); pto->placeholder = state->placeholder; - if (new_nargs && Py_Is(PyTuple_GET_ITEM(args, new_nargs), pto->placeholder)) { + if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == pto->placeholder) { PyErr_SetString(PyExc_TypeError, "trailing Placeholders are not allowed"); return NULL; @@ -214,7 +214,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) } } /* merge args with args of `func` which is `partial` */ - if ((pto_phcount > 0) && (new_nargs > 0)) { + if (pto_phcount > 0 && new_nargs > 0) { Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args); Py_ssize_t tot_nargs = npargs; if (new_nargs > pto_phcount) { @@ -409,7 +409,7 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, tot_nargs = pto_nargs + nargs - pto_phcount; Py_ssize_t j = 0; // New args index for (Py_ssize_t i = 0; i < pto_nargs; i++) { - if (Py_Is(pto_args[i], pto->placeholder)){ + if (pto_args[i] == pto->placeholder){ stack[i] = args[j]; j += 1; } @@ -508,19 +508,19 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) Py_ssize_t j = 0; // New args index for (Py_ssize_t i = 0; i < pto_nargs; i++) { item = PyTuple_GET_ITEM(pto_args, i); - if (Py_Is(item, pto->placeholder)) { + if (item == pto->placeholder) { item = PyTuple_GET_ITEM(args, j); j += 1; } + Py_INCREF(item); PyTuple_SET_ITEM(tot_args, i, item); } assert(j == pto_phcount); - if (nargs > pto_phcount) { - for (Py_ssize_t i = pto_nargs; i < tot_nargs; i++) { - item = PyTuple_GET_ITEM(args, j); - PyTuple_SET_ITEM(tot_args, i, item); - j += 1; - } + for (Py_ssize_t i = pto_nargs; i < tot_nargs; i++) { + item = PyTuple_GET_ITEM(args, j); + Py_INCREF(item); + PyTuple_SET_ITEM(tot_args, i, item); + j += 1; } } else { @@ -670,7 +670,7 @@ partial_setstate(partialobject *pto, PyObject *state) } Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs); - if (nargs && Py_Is(PyTuple_GET_ITEM(fnargs, nargs - 1), pto->placeholder)) { + if (nargs && PyTuple_GET_ITEM(fnargs, nargs - 1) == pto->placeholder) { PyErr_SetString(PyExc_TypeError, "trailing Placeholders are not allowed"); return NULL; @@ -678,7 +678,7 @@ partial_setstate(partialobject *pto, PyObject *state) /* Count placeholders */ Py_ssize_t phcount = 0; for (Py_ssize_t i = 0; i < nargs - 1; i++) { - if (Py_Is(PyTuple_GET_ITEM(fnargs, i), pto->placeholder)) { + if (PyTuple_GET_ITEM(fnargs, i), pto->placeholder) { phcount++; } } From a6c6ef28039305baceaba521d23816016eaf9a81 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 12 Aug 2024 07:18:52 +0300 Subject: [PATCH 57/70] CR Part 2 --- Lib/functools.py | 5 ++--- Modules/_functoolsmodule.c | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 4181d7ed55b901..2309b8b5434419 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -305,14 +305,13 @@ def _partial_prepare_merger(args): if not nargs: return 0, None order = [] - i, j = 0, nargs - for a in args: + j = nargs + for i, a in enumerate(args): if a is Placeholder: order.append(j) j += 1 else: order.append(i) - i += 1 phcount = j - nargs merger = itemgetter(*order) if phcount else None return phcount, merger diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 251c4dbc86fc8c..933d3d4a7d6a8c 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -47,12 +47,13 @@ get_functools_state(PyObject *module) // The 'Placeholder' singleton indicates which formal positional // parameters are to be bound first when using a 'partial' object. -static PyObject* placeholder_instance; - typedef struct { PyObject_HEAD } placeholderobject; +static inline _functools_state * +get_functools_state_by_type(PyTypeObject *type); + PyDoc_STRVAR(placeholder_doc, "The type of the Placeholder singleton.\n\n" "Used as a placeholder for partial arguments."); @@ -91,10 +92,11 @@ placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "PlaceholderType takes no arguments"); return NULL; } - if (placeholder_instance == NULL) { - placeholder_instance = PyType_GenericNew(type, NULL, NULL); + _functools_state *state = get_functools_state_by_type(type); + if (state->placeholder == NULL) { + state->placeholder = PyType_GenericNew(type, NULL, NULL); } - return placeholder_instance; + return state->placeholder; } static PyType_Slot placeholder_type_slots[] = { @@ -209,7 +211,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) /* Count placeholders */ Py_ssize_t phcount = 0; for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { - if (Py_Is(PyTuple_GET_ITEM(new_args, i), pto->placeholder)) { + if (PyTuple_GET_ITEM(new_args, i) == pto->placeholder) { phcount++; } } @@ -225,7 +227,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { if (i < npargs) { item = PyTuple_GET_ITEM(pto_args, i); - if ((j < new_nargs) && Py_Is(item, pto->placeholder)) { + if (j < new_nargs && item == pto->placeholder) { item = PyTuple_GET_ITEM(new_args, j); j++; pto_phcount--; @@ -678,7 +680,7 @@ partial_setstate(partialobject *pto, PyObject *state) /* Count placeholders */ Py_ssize_t phcount = 0; for (Py_ssize_t i = 0; i < nargs - 1; i++) { - if (PyTuple_GET_ITEM(fnargs, i), pto->placeholder) { + if (PyTuple_GET_ITEM(fnargs, i) == pto->placeholder) { phcount++; } } From 1c8d73e3931727573606082684bc94b856805589 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 12 Aug 2024 07:45:58 +0300 Subject: [PATCH 58/70] remove ignored global var --- Tools/c-analyzer/cpython/ignored.tsv | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index e00c3df0ae91ed..63b640e465ac6b 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -228,7 +228,6 @@ Modules/_decimal/_decimal.c - signal_map_template - Modules/_decimal/_decimal.c - ssize_constants - Modules/_decimal/_decimal.c - INVALID_SIGNALDICT_ERROR_MSG - Modules/_elementtree.c - ExpatMemoryHandler - -Modules/_functoolsmodule.c - placeholder_instance - Modules/_hashopenssl.c - py_hashes - Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - Modules/_hacl/Hacl_Hash_MD5.c - _h0 - From a8bd3ae524f6f2cffcd76ed8f9862cc7c5d22ae8 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 12 Aug 2024 14:40:46 +0300 Subject: [PATCH 59/70] CR changes --- Lib/functools.py | 4 +--- Lib/inspect.py | 13 ++++++++++--- Lib/test/test_functools.py | 10 ++++++++-- Lib/test/test_inspect/test_inspect.py | 6 +++--- Modules/_functoolsmodule.c | 6 ------ 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/Lib/functools.py b/Lib/functools.py index 2309b8b5434419..56a979f2e7c251 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -343,10 +343,8 @@ def _partial_new(cls, func, /, *args, **keywords): if nargs > pto_phcount: tot_args += args[pto_phcount:] phcount, merger = _partial_prepare_merger(tot_args) - elif pto_phcount: # not args and pto_phcount + else: # works for both pto_phcount == 0 and != 0 phcount, merger = pto_phcount, func._merger - else: # not args and not pto_phcount - phcount, merger = 0, None keywords = {**func.keywords, **keywords} func = func.func else: diff --git a/Lib/inspect.py b/Lib/inspect.py index 016b998c1edee6..dca3d287479757 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1933,7 +1933,10 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): # If positional-only parameter is bound by partial, # it effectively disappears from the signature # However, if it is a Placeholder it is not removed - if arg_value is not functools.Placeholder: + # And also looses default value + if arg_value is functools.Placeholder: + new_params[param_name] = param.replace(default=_empty) + else: new_params.pop(param_name) continue @@ -1957,9 +1960,13 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): else: # was passed as a positional argument # Do not pop if it is a Placeholder - # and change kind to positional only + # also change kind to positional only + # and remove default if arg_value is functools.Placeholder: - new_param = param.replace(kind=_POSITIONAL_ONLY) + new_param = param.replace( + kind=_POSITIONAL_ONLY, + default=_empty + ) new_params[param_name] = new_param else: new_params.pop(param_name) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index a86a7f6fbe1c31..770b34d8bf49b8 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -242,6 +242,12 @@ def test_placeholders_optimization(self): got, empty = p3(5) expected = (-1, 0, 1, 2, 3, 4, 5) self.assertTrue(expected == got and empty == {}) + # inner partial has placeholders and outer partial has no args case + p = self.partial(capture, PH, 0) + p2 = self.partial(p) + expected = (PH, 0) + self.assertTrue(expected == p2.args) + self.assertTrue(((1, 0), {}) == p2(1)) def test_construct_placeholder_singleton(self): PH = self.module.Placeholder @@ -550,7 +556,7 @@ class TestPartialPySubclass(TestPartialPy): def test_subclass_optimization(self): p = py_functools.partial(min, 2) p2 = self.partial(p, 1) - assert p2.func == min + assert p2.func is min assert p2(0) == 0 @@ -696,7 +702,7 @@ class PartialMethodSubclass(functools.partialmethod): pass p = functools.partialmethod(min, 2) p2 = PartialMethodSubclass(p, 1) - assert p2.func == min + assert p2.func is min assert p2.__get__(0)() == 0 diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 20c5a1d98590ab..27f38dedcfa41f 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3523,7 +3523,7 @@ def foo(a=0, b=1, /, c=2, d=3): # Positional p = partial(foo, Placeholder, 1, c=0, d=1) self.assertEqual(self.signature(p), - ((('a', 0, ..., "positional_only"), + ((('a', ..., ..., "positional_only"), ('c', 0, ..., "keyword_only"), ('d', 1, ..., "keyword_only")), ...)) @@ -3531,8 +3531,8 @@ def foo(a=0, b=1, /, c=2, d=3): # Positional or Keyword - transformed to positional p = partial(foo, Placeholder, 1, Placeholder, 1) self.assertEqual(self.signature(p), - ((('a', 0, ..., "positional_only"), - ('c', 2, ..., "positional_only")), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "positional_only")), ...)) def test_signature_on_partialmethod(self): diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 933d3d4a7d6a8c..5577c386922bcc 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -656,12 +656,6 @@ partial_setstate(partialobject *pto, PyObject *state) PyErr_SetString(PyExc_TypeError, "invalid partial state"); return NULL; } - Py_ssize_t state_len = PyTuple_GET_SIZE(state); - if (state_len != 4) { - PyErr_Format(PyExc_TypeError, - "expected 4 items in state, got %zd", state_len); - return NULL; - } if (!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || !PyCallable_Check(fn) || !PyTuple_Check(fnargs) || From 70e47ed3981ecb38b60e91892a2df3c3fca56af6 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Mon, 12 Aug 2024 15:18:19 +0300 Subject: [PATCH 60/70] small CR changes and doc updates --- Doc/library/functools.rst | 22 +++++++++++++--------- Lib/test/test_functools.py | 8 ++++---- Modules/_functoolsmodule.c | 8 ++++---- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 7088130cdb6763..393394c95c484b 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -373,20 +373,24 @@ The :mod:`functools` module defines the following functions: only one positional argument is provided, while there are two placeholders in :ref:`partial object `. - When successively using :func:`partial` existing :data:`!Placeholder` - sentinels are filled first. A place for positional argument is retained - when :data:`!Placeholder` sentinel is replaced with a new one: + Successive :func:`partial` applications fill :data:`!Placeholder` sentinels + of the input :func:`partial` objects with new positional arguments. + A place for positional argument can be retained by inserting new + :data:`!Placeholder` sentinel to the place held by previous :data:`!Placeholder`: >>> from functools import partial, Placeholder as _ - >>> count = partial(print, _, _, _, 4) - >>> count = partial(count, _, _, 3) - >>> count = partial(count, _, 2) - >>> count = partial(count, _, 5) # 5 is appended after 4 - >>> count(1) + >>> show5 = partial(print, _, _, _, 4) + >>> show5 = partial(show5, _, _, 3) + >>> show5 = partial(show5, _, 2) + >>> show5 = partial(show5, _, 5) # 5 is appended after 4 + >>> show5(1) 1 2 3 4 5 + Note, :data:`!Placeholder` has no special treatment when used for keyword + argument of :data:`!Placeholder`. + .. versionchanged:: 3.14 - Support for :data:`Placeholder` in *args* + Added support for :data:`Placeholder` in positional arguments. .. data:: Placeholder diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 770b34d8bf49b8..9cc4ec4d5c635a 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -556,8 +556,8 @@ class TestPartialPySubclass(TestPartialPy): def test_subclass_optimization(self): p = py_functools.partial(min, 2) p2 = self.partial(p, 1) - assert p2.func is min - assert p2(0) == 0 + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) class TestPartialMethod(unittest.TestCase): @@ -702,8 +702,8 @@ class PartialMethodSubclass(functools.partialmethod): pass p = functools.partialmethod(min, 2) p2 = PartialMethodSubclass(p, 1) - assert p2.func is min - assert p2.__get__(0)() == 0 + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) class TestUpdateWrapper(unittest.TestCase): diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 5577c386922bcc..42dfc9dfb1b515 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -150,14 +150,13 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) PyObject *func, *pto_args, *new_args, *pto_kw; partialobject *pto; Py_ssize_t pto_phcount = 0; - Py_ssize_t new_nargs = PyTuple_GET_SIZE(args); + Py_ssize_t new_nargs = PyTuple_GET_SIZE(args) - 1; - if (new_nargs < 1) { + if (new_nargs < 0) { PyErr_SetString(PyExc_TypeError, "type 'partial' takes at least one argument"); return NULL; } - new_nargs--; func = PyTuple_GET_ITEM(args, 0); if (!PyCallable_Check(func)) { PyErr_SetString(PyExc_TypeError, @@ -411,7 +410,7 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, tot_nargs = pto_nargs + nargs - pto_phcount; Py_ssize_t j = 0; // New args index for (Py_ssize_t i = 0; i < pto_nargs; i++) { - if (pto_args[i] == pto->placeholder){ + if (pto_args[i] == pto->placeholder) { stack[i] = args[j]; j += 1; } @@ -500,6 +499,7 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs) if (pto_phcount) { Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); Py_ssize_t tot_nargs = pto_nargs + nargs - pto_phcount; + assert(tot_nargs >= 0); tot_args = PyTuple_New(tot_nargs); if (tot_args == NULL) { Py_XDECREF(tot_kw); From 2eacf5e40d24a8cf988993db3aaa7725b0c15ac6 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 13 Aug 2024 02:50:33 +0300 Subject: [PATCH 61/70] push placeholder check to earlier place --- Modules/_functoolsmodule.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 42dfc9dfb1b515..2b3bd7c3de1176 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -147,7 +147,7 @@ get_functools_state_by_type(PyTypeObject *type) static PyObject * partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *func, *pto_args, *new_args, *pto_kw; + PyObject *func, *pto_args, *new_args, *pto_kw, *phold; partialobject *pto; Py_ssize_t pto_phcount = 0; Py_ssize_t new_nargs = PyTuple_GET_SIZE(args) - 1; @@ -168,7 +168,16 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (state == NULL) { return NULL; } + phold = state->placeholder; + /* Placeholder restrictions */ + if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == phold) { + PyErr_SetString(PyExc_TypeError, + "trailing Placeholders are not allowed"); + return NULL; + } + + /* check wrapped function / object */ pto_args = pto_kw = NULL; int res = PyObject_TypeCheck(func, state->partial_type); if (res == -1) { @@ -193,13 +202,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; pto->fn = Py_NewRef(func); - - pto->placeholder = state->placeholder; - if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == pto->placeholder) { - PyErr_SetString(PyExc_TypeError, - "trailing Placeholders are not allowed"); - return NULL; - } + pto->placeholder = phold; new_args = PyTuple_GetSlice(args, 1, new_nargs + 1); if (new_args == NULL) { @@ -210,7 +213,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) /* Count placeholders */ Py_ssize_t phcount = 0; for (Py_ssize_t i = 0; i < new_nargs - 1; i++) { - if (PyTuple_GET_ITEM(new_args, i) == pto->placeholder) { + if (PyTuple_GET_ITEM(new_args, i) == phold) { phcount++; } } @@ -226,7 +229,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) { if (i < npargs) { item = PyTuple_GET_ITEM(pto_args, i); - if (j < new_nargs && item == pto->placeholder) { + if (j < new_nargs && item == phold) { item = PyTuple_GET_ITEM(new_args, j); j++; pto_phcount--; From f78d8d31cada45ad47d6efdf8d48154726584772 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 13 Aug 2024 17:24:18 +0300 Subject: [PATCH 62/70] more appropriate test functions and better doc example --- Doc/library/functools.rst | 15 +++++++++------ Lib/test/test_functools.py | 14 ++++++++------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 393394c95c484b..0e4282e4d5576a 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -379,12 +379,15 @@ The :mod:`functools` module defines the following functions: :data:`!Placeholder` sentinel to the place held by previous :data:`!Placeholder`: >>> from functools import partial, Placeholder as _ - >>> show5 = partial(print, _, _, _, 4) - >>> show5 = partial(show5, _, _, 3) - >>> show5 = partial(show5, _, 2) - >>> show5 = partial(show5, _, 5) # 5 is appended after 4 - >>> show5(1) - 1 2 3 4 5 + >>> remove = partial(str.replace, _, _, '') + >>> remove('Hello, world', 'l') + 'Heo, word' + >>> remove_l = partial(remove, _, 'l') + >>> remove_l('Hello, world') + 'Heo, word' + >>> remove_first_l = partial(remove_l, _, 1) + >>> remove_first_l('Hello, world') + 'Helo, world' Note, :data:`!Placeholder` has no special treatment when used for keyword argument of :data:`!Placeholder`. diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 9cc4ec4d5c635a..ca7c5e6165cf52 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -222,22 +222,24 @@ def test_placeholders(self): args = (PH, 0) p = self.partial(capture, *args) got, empty = p('x') - self.assertTrue(('x', 0) == got and empty == {}) + self.assertEqual(('x', 0), got) + self.assertEqual(empty, {}) # 2 Placeholders args = (PH, 0, PH, 1) p = self.partial(capture, *args) with self.assertRaises(TypeError): - got, empty = p('x') + p('x') got, empty = p('x', 'y') expected = ('x', 0, 'y', 1) - self.assertTrue(expected == got and empty == {}) + self.assertEqual(expected, got) + self.assertEqual(empty, {}) def test_placeholders_optimization(self): PH = self.module.Placeholder p = self.partial(capture, PH, 0) p2 = self.partial(p, PH, 1, 2, 3) expected = (PH, 0, 1, 2, 3) - self.assertTrue(expected == p2.args) + self.assertEqual(expected, p2.args) p3 = self.partial(p2, -1, 4) got, empty = p3(5) expected = (-1, 0, 1, 2, 3, 4, 5) @@ -246,8 +248,8 @@ def test_placeholders_optimization(self): p = self.partial(capture, PH, 0) p2 = self.partial(p) expected = (PH, 0) - self.assertTrue(expected == p2.args) - self.assertTrue(((1, 0), {}) == p2(1)) + self.assertEqual(expected, p2.args) + self.assertEqual(((1, 0), {}), p2(1)) def test_construct_placeholder_singleton(self): PH = self.module.Placeholder From 0a8640e400140fefd12d68d8599e8f449b3f03dd Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 13 Aug 2024 18:14:34 +0300 Subject: [PATCH 63/70] minor fixes; message, doc polish --- Doc/library/functools.rst | 4 ++-- Doc/whatsnew/3.14.rst | 16 ++++++++-------- Lib/functools.py | 10 +++++----- Lib/inspect.py | 2 +- Lib/test/test_functools.py | 23 ++++++++++++++++++++--- 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 0e4282e4d5576a..d987c24d528645 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -348,7 +348,7 @@ The :mod:`functools` module defines the following functions: The :func:`partial` is used for partial function application which "freezes" some portion of a function's arguments and/or keywords resulting in a new object - with a simplified signature. For example, :func:`partial` can be used to create + with a simplified signature. For example, :func:`partial` can be used to create a callable that behaves like the :func:`int` function where the *base* argument defaults to two: @@ -362,7 +362,7 @@ The :mod:`functools` module defines the following functions: when :func:`partial` is called. This allows custom selection of positional arguments to be pre-filled when constructing a :ref:`partial object `. - If :data:`!Placeholder` sentinels are used, all of them must be filled at call time: + If :data:`!Placeholder` sentinels are present, all of them must be filled at call time: >>> from functools import partial, Placeholder >>> say_to_world = partial(print, Placeholder, Placeholder, "world!") diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index e0420afaca7348..e753a2c535a283 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -117,6 +117,14 @@ Added support for converting any objects that have the :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. (Contributed by Serhiy Storchaka in :gh:`82017`.) +functools +--------- + +* Added support to :func:`functools.partial` and + :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels + to reserve a place for positional arguments. + (Contributed by Dominykas Grigonis in :gh:`119127`.) + json ---- @@ -133,14 +141,6 @@ operator to ``obj is not None``. (Contributed by Raymond Hettinger and Nico Mexis in :gh:`115808`.) -functools ---------- - -* :func:`functools.partial` and :func:`functools.partialmethod` now support - :data:`functools.Placeholder` sentinels to reserve a place for - positional arguments. - (Contributed by Dominykas Grigonis in :gh:`119127`) - os -- diff --git a/Lib/functools.py b/Lib/functools.py index 56a979f2e7c251..9de95943dd181b 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -301,9 +301,9 @@ def __reduce__(self): Placeholder = _PlaceholderType() def _partial_prepare_merger(args): - nargs = len(args) - if not nargs: + if not args: return 0, None + nargs = len(args) order = [] j = nargs for i, a in enumerate(args): @@ -324,9 +324,9 @@ def _partial_new(cls, func, /, *args, **keywords): else: base_cls = partialmethod # func could be a descriptor like classmethod which isn't callable - # assert issubclass(cls, partialmethod) if not callable(func) and not hasattr(func, "__get__"): - raise TypeError(f"{func!r} is not callable or a descriptor") + raise TypeError(f"the first argument {func!r} must be a callable " + "or a descriptor") if args and args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") if isinstance(func, base_cls): @@ -459,7 +459,7 @@ def _method(cls_or_self, /, *args, **keywords): args = args[phcount:] except IndexError: raise TypeError("missing positional arguments " - "in 'partial' call; expected " + "in 'partialmethod' call; expected " f"at least {phcount}, got {len(args)}") else: pto_args = self.args diff --git a/Lib/inspect.py b/Lib/inspect.py index dca3d287479757..15de1bc1917ca0 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2464,7 +2464,7 @@ def _signature_from_callable(obj, *, assert (not sig_params or first_wrapped_param is not sig_params[0]) # If there were placeholders set, - # first param is transformaed to positional only + # first param is transformed to positional only if partialmethod.args.count(functools.Placeholder): first_wrapped_param = first_wrapped_param.replace( kind=Parameter.POSITIONAL_ONLY) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index ca7c5e6165cf52..fd8beda37a3092 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -364,14 +364,19 @@ def test_setstate(self): f = self.partial(signature) f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[]))) self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[]))) - with self.assertRaises(TypeError): + with self.assertRaises(TypeError) as cm: self.assertEqual(f(), (PH, 1), dict(a=10)) + expected = ("missing positional arguments in 'partial' call; " + "expected at least 1, got 0") + self.assertEqual(cm.exception.args[0], expected) self.assertEqual(f(2), ((2, 1), dict(a=10))) - # Leading Placeholder error + # Trailing Placeholder error f = self.partial(signature) - with self.assertRaises(TypeError): + with self.assertRaises(TypeError) as cm: f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) + expected = "trailing Placeholders are not allowed" + self.assertEqual(cm.exception.args[0], expected) def test_setstate_errors(self): f = self.partial(signature) @@ -556,10 +561,16 @@ class TestPartialPySubclass(TestPartialPy): partial = PyPartialSubclass def test_subclass_optimization(self): + # `partial` input to `partial` subclass p = py_functools.partial(min, 2) p2 = self.partial(p, 1) self.assertIs(p2.func, min) self.assertEqual(p2(0), 0) + # `partial` subclass input to `partial` subclass + p = self.partial(min, 2) + p2 = self.partial(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2(0), 0) class TestPartialMethod(unittest.TestCase): @@ -702,10 +713,16 @@ def f(a, b, /): def test_subclass_optimization(self): class PartialMethodSubclass(functools.partialmethod): pass + # `partialmethod` input to `partialmethod` subclass p = functools.partialmethod(min, 2) p2 = PartialMethodSubclass(p, 1) self.assertIs(p2.func, min) self.assertEqual(p2.__get__(0)(), 0) + # `partialmethod` subclass input to `partialmethod` subclass + p = PartialMethodSubclass(min, 2) + p2 = PartialMethodSubclass(p, 1) + self.assertIs(p2.func, min) + self.assertEqual(p2.__get__(0)(), 0) class TestUpdateWrapper(unittest.TestCase): From 6e3d28217e1e80af1a32cb08dbc9e0908fa9ff99 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Tue, 13 Aug 2024 20:46:51 +0300 Subject: [PATCH 64/70] better doc example and small test changes --- Doc/library/functools.rst | 17 ++++++++-------- Lib/test/test_functools.py | 41 +++++++++++++++++--------------------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index d987c24d528645..563090152cd095 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -380,14 +380,15 @@ The :mod:`functools` module defines the following functions: >>> from functools import partial, Placeholder as _ >>> remove = partial(str.replace, _, _, '') - >>> remove('Hello, world', 'l') - 'Heo, word' - >>> remove_l = partial(remove, _, 'l') - >>> remove_l('Hello, world') - 'Heo, word' - >>> remove_first_l = partial(remove_l, _, 1) - >>> remove_first_l('Hello, world') - 'Helo, world' + >>> message = 'Hello, dear dear world!' + >>> remove(message, ' dear') + 'Hello, world!' + >>> remove_dear = partial(remove, _, ' dear') + >>> remove_dear(message) + 'Hello, world!' + >>> remove_first_dear = partial(remove_dear, _, 1) + >>> remove_first_dear(message) + 'Hello, dear world!' Note, :data:`!Placeholder` has no special treatment when used for keyword argument of :data:`!Placeholder`. diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index fd8beda37a3092..26a173e3492c65 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -221,35 +221,32 @@ def test_placeholders(self): # 1 Placeholder args = (PH, 0) p = self.partial(capture, *args) - got, empty = p('x') - self.assertEqual(('x', 0), got) - self.assertEqual(empty, {}) + actual_args, actual_kwds = p('x') + self.assertEqual(actual_args, ('x', 0)) + self.assertEqual(actual_kwds, {}) # 2 Placeholders args = (PH, 0, PH, 1) p = self.partial(capture, *args) with self.assertRaises(TypeError): p('x') - got, empty = p('x', 'y') - expected = ('x', 0, 'y', 1) - self.assertEqual(expected, got) - self.assertEqual(empty, {}) + actual_args, actual_kwds = p('x', 'y') + self.assertEqual(actual_args, ('x', 0, 'y', 1)) + self.assertEqual(actual_kwds, {}) def test_placeholders_optimization(self): PH = self.module.Placeholder p = self.partial(capture, PH, 0) p2 = self.partial(p, PH, 1, 2, 3) - expected = (PH, 0, 1, 2, 3) - self.assertEqual(expected, p2.args) + self.assertEqual(p2.args, (PH, 0, 1, 2, 3)) p3 = self.partial(p2, -1, 4) - got, empty = p3(5) - expected = (-1, 0, 1, 2, 3, 4, 5) - self.assertTrue(expected == got and empty == {}) + actual_args, actual_kwds = p3(5) + self.assertEqual(actual_args, (-1, 0, 1, 2, 3, 4, 5)) + self.assertEqual(actual_kwds, {}) # inner partial has placeholders and outer partial has no args case p = self.partial(capture, PH, 0) p2 = self.partial(p) - expected = (PH, 0) - self.assertEqual(expected, p2.args) - self.assertEqual(((1, 0), {}), p2(1)) + self.assertEqual(p2.args, (PH, 0)) + self.assertEqual(p2(1), ((1, 0), {})) def test_construct_placeholder_singleton(self): PH = self.module.Placeholder @@ -364,19 +361,17 @@ def test_setstate(self): f = self.partial(signature) f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[]))) self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[]))) - with self.assertRaises(TypeError) as cm: - self.assertEqual(f(), (PH, 1), dict(a=10)) - expected = ("missing positional arguments in 'partial' call; " - "expected at least 1, got 0") - self.assertEqual(cm.exception.args[0], expected) + msg_regex = ("^missing positional arguments in 'partial' call; " + "expected at least 1, got 0$") + with self.assertRaisesRegex(TypeError, msg_regex) as cm: + f() self.assertEqual(f(2), ((2, 1), dict(a=10))) # Trailing Placeholder error f = self.partial(signature) - with self.assertRaises(TypeError) as cm: + msg_regex = "^trailing Placeholders are not allowed$" + with self.assertRaisesRegex(TypeError, msg_regex) as cm: f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) - expected = "trailing Placeholders are not allowed" - self.assertEqual(cm.exception.args[0], expected) def test_setstate_errors(self): f = self.partial(signature) From 66c305d1d196e7eee63a4047ef2e99fc7ec5e03d Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 14 Aug 2024 18:31:25 +0300 Subject: [PATCH 65/70] assertRaisesRegex exact match --- Lib/test/test_functools.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 26a173e3492c65..bdaa9a7ec4f020 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -6,6 +6,7 @@ from itertools import permutations import pickle from random import choice +import re import sys from test import support import threading @@ -361,16 +362,16 @@ def test_setstate(self): f = self.partial(signature) f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[]))) self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[]))) - msg_regex = ("^missing positional arguments in 'partial' call; " - "expected at least 1, got 0$") - with self.assertRaisesRegex(TypeError, msg_regex) as cm: + msg_regex = re.escape("missing positional arguments in 'partial' call; " + "expected at least 1, got 0") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: f() self.assertEqual(f(2), ((2, 1), dict(a=10))) # Trailing Placeholder error f = self.partial(signature) - msg_regex = "^trailing Placeholders are not allowed$" - with self.assertRaisesRegex(TypeError, msg_regex) as cm: + msg_regex = re.escape("trailing Placeholders are not allowed") + with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm: f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[]))) def test_setstate_errors(self): From 14bf68c975a94db70bece989aef229a81bbb8a01 Mon Sep 17 00:00:00 2001 From: "d.grigonis" Date: Wed, 14 Aug 2024 23:09:12 +0300 Subject: [PATCH 66/70] doc nits --- Doc/library/functools.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 563090152cd095..de3036340e6bd7 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -328,6 +328,14 @@ The :mod:`functools` module defines the following functions: Returning ``NotImplemented`` from the underlying comparison function for unrecognised types is now supported. +.. data:: Placeholder + + A singleton object used as a sentinel to reserve a place + for positional arguments when calling :func:`partial` + and :func:`partialmethod`. + + .. versionadded:: 3.14 + .. function:: partial(func, /, *args, **keywords) Return a new :ref:`partial object` which when called @@ -348,7 +356,7 @@ The :mod:`functools` module defines the following functions: The :func:`partial` is used for partial function application which "freezes" some portion of a function's arguments and/or keywords resulting in a new object - with a simplified signature. For example, :func:`partial` can be used to create + with a simplified signature. For example, :func:`partial` can be used to create a callable that behaves like the :func:`int` function where the *base* argument defaults to two: @@ -396,12 +404,6 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.14 Added support for :data:`Placeholder` in positional arguments. -.. data:: Placeholder - - A singleton object used as a sentinel to reserve a place - for positional arguments when calling :func:`partial` - and :func:`partialmethod`. - .. class:: partialmethod(func, /, *args, **keywords) Return a new :class:`partialmethod` descriptor which behaves From ee642d57039f5379fbc5d69f7e95468a91160b0c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 25 Sep 2024 12:20:21 -0700 Subject: [PATCH 67/70] Add Placeholder to __all__ --- Lib/functools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/functools.py b/Lib/functools.py index 9de95943dd181b..1e4981dee1c298 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -12,7 +12,7 @@ __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', - 'cached_property'] + 'cached_property', 'Placeholder'] from abc import get_cache_token from collections import namedtuple From 8d6c28ed4fb41251a3afdfa62f982a858cc30ebc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 25 Sep 2024 12:24:03 -0700 Subject: [PATCH 68/70] Update copyright span --- Lib/functools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/functools.py b/Lib/functools.py index 1e4981dee1c298..83b8895794e7c0 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -6,7 +6,7 @@ # Written by Nick Coghlan , # Raymond Hettinger , # and Łukasz Langa . -# Copyright (C) 2006-2013 Python Software Foundation. +# Copyright (C) 2006-2024 Python Software Foundation. # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', From 8744bcbb797750532e0b9851e805757d2e89cf89 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 25 Sep 2024 13:58:15 -0700 Subject: [PATCH 69/70] Minor doc edits and add doctests --- Doc/library/functools.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index de3036340e6bd7..e93e30312d55f8 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -346,21 +346,21 @@ The :mod:`functools` module defines the following functions: Roughly equivalent to:: def partial(func, /, *args, **keywords): - def newfunc(*fargs, **fkeywords): - newkeywords = {**keywords, **fkeywords} - return func(*args, *fargs, **newkeywords) + def newfunc(*more_args, **more_keywords): + return func(*args, *more_args, **keywords, **more_keywords) newfunc.func = func newfunc.args = args newfunc.keywords = keywords return newfunc - The :func:`partial` is used for partial function application which "freezes" + The :func:`partial` function is used for partial function application which "freezes" some portion of a function's arguments and/or keywords resulting in a new object with a simplified signature. For example, :func:`partial` can be used to create a callable that behaves like the :func:`int` function where the *base* argument - defaults to two: + defaults to ``2``: + + .. doctest:: - >>> from functools import partial >>> basetwo = partial(int, base=2) >>> basetwo.__doc__ = 'Convert base 2 string to an int.' >>> basetwo('10010') @@ -372,7 +372,8 @@ The :mod:`functools` module defines the following functions: If :data:`!Placeholder` sentinels are present, all of them must be filled at call time: - >>> from functools import partial, Placeholder + .. doctest:: + >>> say_to_world = partial(print, Placeholder, Placeholder, "world!") >>> say_to_world('Hello', 'dear') Hello dear world! @@ -386,6 +387,8 @@ The :mod:`functools` module defines the following functions: A place for positional argument can be retained by inserting new :data:`!Placeholder` sentinel to the place held by previous :data:`!Placeholder`: + .. doctest:: + >>> from functools import partial, Placeholder as _ >>> remove = partial(str.replace, _, _, '') >>> message = 'Hello, dear dear world!' @@ -789,6 +792,4 @@ have three read-only attributes: :class:`partial` objects are like :class:`function` objects in that they are callable, weak referenceable, and can have attributes. There are some important differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes -are not created automatically. Also, :class:`partial` objects defined in -classes behave like static methods and do not transform into bound methods -during instance attribute look-up. +are not created automatically. From c3ad7d9b39f20accdc1f5c89936ea3618b688f4b Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 25 Sep 2024 17:23:25 -0700 Subject: [PATCH 70/70] Having the separate dict was necessary to eliminate duplicate keywords --- Doc/library/functools.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index e9a08af704069a..774b3262117723 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -347,7 +347,8 @@ The :mod:`functools` module defines the following functions: def partial(func, /, *args, **keywords): def newfunc(*more_args, **more_keywords): - return func(*args, *more_args, **keywords, **more_keywords) + keywords_union = {**keywords, **more_keywords} + return func(*args, *more_args, **keywords_union) newfunc.func = func newfunc.args = args newfunc.keywords = keywords