From c28c79fbc2245ab5016e6feafbbc9002cef9e11b Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sat, 2 Mar 2024 16:03:38 +0900 Subject: [PATCH 1/5] gh-115103: Update refleak checker to trigger _PyMem_ProcessDelayed --- Include/internal/pycore_gc.h | 3 +++ Lib/test/support/__init__.py | 1 + Modules/clinic/gcmodule.c.h | 20 +++++++++++++++++++- Modules/gcmodule.c | 19 +++++++++++++++++++ Python/gc_free_threading.c | 12 ++++++++++++ 5 files changed, 54 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 40414a868518bb..dfce27a9c4c6fa 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -288,6 +288,9 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); +#ifdef Py_GIL_DISABLED +extern void _PyGC_Clear_DelayedObjects(PyInterpreterState *interp); +#endif #ifdef __cplusplus } diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 14e3766a34377b..97bce67e7c37d5 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -785,6 +785,7 @@ def gc_collect(): gc.collect() gc.collect() gc.collect() + gc._collect_delayed_objects() @contextlib.contextmanager def disable_gc(): diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 9fff4da616ba00..a91fd630b46d6f 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -147,6 +147,24 @@ gc_collect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * return return_value; } +PyDoc_STRVAR(gc__collect_delayed_objects__doc__, +"_collect_delayed_objects($module, /)\n" +"--\n" +"\n" +"Process delayed free requests by force"); + +#define GC__COLLECT_DELAYED_OBJECTS_METHODDEF \ + {"_collect_delayed_objects", (PyCFunction)gc__collect_delayed_objects, METH_NOARGS, gc__collect_delayed_objects__doc__}, + +static PyObject * +gc__collect_delayed_objects_impl(PyObject *module); + +static PyObject * +gc__collect_delayed_objects(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return gc__collect_delayed_objects_impl(module); +} + PyDoc_STRVAR(gc_set_debug__doc__, "set_debug($module, flags, /)\n" "--\n" @@ -585,4 +603,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=0a7e91917adcb937 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6cda64b0da33bcab input=a9049054013a1b77]*/ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 9807d2e7d48a36..bb9d7b5b1f7149 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -93,6 +93,24 @@ gc_collect_impl(PyObject *module, int generation) return _PyGC_Collect(tstate, generation, _Py_GC_REASON_MANUAL); } +/*[clinic input] +gc._collect_delayed_objects + +Process delayed free requests by force + +[clinic start generated code]*/ + +static PyObject * +gc__collect_delayed_objects_impl(PyObject *module) +/*[clinic end generated code: output=a016a10f967d4229 input=1064c31903cd9fac]*/ +{ +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyGC_Clear_DelayedObjects(interp); +#endif + Py_RETURN_NONE; +} + /*[clinic input] gc.set_debug @@ -508,6 +526,7 @@ static PyMethodDef GcMethods[] = { GC_FREEZE_METHODDEF GC_UNFREEZE_METHODDEF GC_GET_FREEZE_COUNT_METHODDEF + GC__COLLECT_DELAYED_OBJECTS_METHODDEF {NULL, NULL} /* Sentinel */ }; diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index d4fb50106093ee..70871eb3b6cc15 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1758,4 +1758,16 @@ _PyGC_ClearAllFreeLists(PyInterpreterState *interp) HEAD_UNLOCK(&_PyRuntime); } +void +_PyGC_Clear_DelayedObjects(PyInterpreterState *interp) +{ + HEAD_LOCK(&_PyRuntime); + PyThreadState *tstate = interp->threads.head; + while (tstate != NULL) { + _PyMem_ProcessDelayed(tstate); + tstate = (PyThreadState *)tstate->next; + } + HEAD_UNLOCK(&_PyRuntime); +} + #endif // Py_GIL_DISABLED From 81f333a4bb011b9432fc504863e8667383e09f2d Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sat, 2 Mar 2024 17:56:26 +0900 Subject: [PATCH 2/5] Update Lib/test/support/__init__.py --- Lib/test/support/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 97bce67e7c37d5..dcd2910835f0bc 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -787,6 +787,7 @@ def gc_collect(): gc.collect() gc._collect_delayed_objects() + @contextlib.contextmanager def disable_gc(): import gc From 59b3fc4808b1470211d3d362eee5261e842c6099 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sat, 2 Mar 2024 22:18:21 +0900 Subject: [PATCH 3/5] Address code review --- Lib/test/support/__init__.py | 3 ++- Modules/clinic/gcmodule.c.h | 10 +++++++++- Modules/gcmodule.c | 4 ++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index dcd2910835f0bc..3d8d6715355bd2 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -785,7 +785,8 @@ def gc_collect(): gc.collect() gc.collect() gc.collect() - gc._collect_delayed_objects() + if Py_GIL_DISABLED: + gc._collect_delayed_objects() @contextlib.contextmanager diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index a91fd630b46d6f..50004c5a05d6c9 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -147,6 +147,8 @@ gc_collect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * return return_value; } +#if defined(Py_GIL_DISABLED) + PyDoc_STRVAR(gc__collect_delayed_objects__doc__, "_collect_delayed_objects($module, /)\n" "--\n" @@ -165,6 +167,8 @@ gc__collect_delayed_objects(PyObject *module, PyObject *Py_UNUSED(ignored)) return gc__collect_delayed_objects_impl(module); } +#endif /* defined(Py_GIL_DISABLED) */ + PyDoc_STRVAR(gc_set_debug__doc__, "set_debug($module, flags, /)\n" "--\n" @@ -603,4 +607,8 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=6cda64b0da33bcab input=a9049054013a1b77]*/ + +#ifndef GC__COLLECT_DELAYED_OBJECTS_METHODDEF + #define GC__COLLECT_DELAYED_OBJECTS_METHODDEF +#endif /* !defined(GC__COLLECT_DELAYED_OBJECTS_METHODDEF) */ +/*[clinic end generated code: output=1f2c42da0ff00087 input=a9049054013a1b77]*/ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index bb9d7b5b1f7149..41afbb36881348 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -93,6 +93,7 @@ gc_collect_impl(PyObject *module, int generation) return _PyGC_Collect(tstate, generation, _Py_GC_REASON_MANUAL); } +#ifdef Py_GIL_DISABLED /*[clinic input] gc._collect_delayed_objects @@ -104,12 +105,11 @@ static PyObject * gc__collect_delayed_objects_impl(PyObject *module) /*[clinic end generated code: output=a016a10f967d4229 input=1064c31903cd9fac]*/ { -#ifdef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); _PyGC_Clear_DelayedObjects(interp); -#endif Py_RETURN_NONE; } +#endif /*[clinic input] gc.set_debug From 85b419c2b0107e5be77d87fd314ae3213d805753 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 3 Mar 2024 02:32:01 +0900 Subject: [PATCH 4/5] Address code review --- Include/internal/pycore_gc.h | 3 --- Lib/test/support/__init__.py | 2 -- Modules/clinic/gcmodule.c.h | 28 +--------------------------- Modules/gcmodule.c | 19 ------------------- Python/gc_free_threading.c | 30 ++++++++++++++++++------------ 5 files changed, 19 insertions(+), 63 deletions(-) diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index dfce27a9c4c6fa..40414a868518bb 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -288,9 +288,6 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); -#ifdef Py_GIL_DISABLED -extern void _PyGC_Clear_DelayedObjects(PyInterpreterState *interp); -#endif #ifdef __cplusplus } diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 3d8d6715355bd2..aa6f71d0aad3cc 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -785,8 +785,6 @@ def gc_collect(): gc.collect() gc.collect() gc.collect() - if Py_GIL_DISABLED: - gc._collect_delayed_objects() @contextlib.contextmanager diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 50004c5a05d6c9..9fff4da616ba00 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -147,28 +147,6 @@ gc_collect(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * return return_value; } -#if defined(Py_GIL_DISABLED) - -PyDoc_STRVAR(gc__collect_delayed_objects__doc__, -"_collect_delayed_objects($module, /)\n" -"--\n" -"\n" -"Process delayed free requests by force"); - -#define GC__COLLECT_DELAYED_OBJECTS_METHODDEF \ - {"_collect_delayed_objects", (PyCFunction)gc__collect_delayed_objects, METH_NOARGS, gc__collect_delayed_objects__doc__}, - -static PyObject * -gc__collect_delayed_objects_impl(PyObject *module); - -static PyObject * -gc__collect_delayed_objects(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return gc__collect_delayed_objects_impl(module); -} - -#endif /* defined(Py_GIL_DISABLED) */ - PyDoc_STRVAR(gc_set_debug__doc__, "set_debug($module, flags, /)\n" "--\n" @@ -607,8 +585,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } - -#ifndef GC__COLLECT_DELAYED_OBJECTS_METHODDEF - #define GC__COLLECT_DELAYED_OBJECTS_METHODDEF -#endif /* !defined(GC__COLLECT_DELAYED_OBJECTS_METHODDEF) */ -/*[clinic end generated code: output=1f2c42da0ff00087 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0a7e91917adcb937 input=a9049054013a1b77]*/ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 41afbb36881348..9807d2e7d48a36 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -93,24 +93,6 @@ gc_collect_impl(PyObject *module, int generation) return _PyGC_Collect(tstate, generation, _Py_GC_REASON_MANUAL); } -#ifdef Py_GIL_DISABLED -/*[clinic input] -gc._collect_delayed_objects - -Process delayed free requests by force - -[clinic start generated code]*/ - -static PyObject * -gc__collect_delayed_objects_impl(PyObject *module) -/*[clinic end generated code: output=a016a10f967d4229 input=1064c31903cd9fac]*/ -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyGC_Clear_DelayedObjects(interp); - Py_RETURN_NONE; -} -#endif - /*[clinic input] gc.set_debug @@ -526,7 +508,6 @@ static PyMethodDef GcMethods[] = { GC_FREEZE_METHODDEF GC_UNFREEZE_METHODDEF GC_GET_FREEZE_COUNT_METHODDEF - GC__COLLECT_DELAYED_OBJECTS_METHODDEF {NULL, NULL} /* Sentinel */ }; diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 70871eb3b6cc15..18893c6c391fff 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -324,6 +324,23 @@ merge_all_queued_objects(PyInterpreterState *interp, struct collection_state *st HEAD_UNLOCK(&_PyRuntime); } +static void +process_delayed_frees(PyInterpreterState *interp) +{ + // In STW status, we can observe the latest write sequence by + // advancing the write sequence immediately. + _Py_qsbr_advance(&interp->qsbr); + _PyThreadStateImpl *current_tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + _Py_qsbr_quiescent_state(current_tstate->qsbr); + HEAD_LOCK(&_PyRuntime); + PyThreadState *tstate = interp->threads.head; + while (tstate != NULL) { + _PyMem_ProcessDelayed(tstate); + tstate = (PyThreadState *)tstate->next; + } + HEAD_UNLOCK(&_PyRuntime); +} + // Subtract an incoming reference from the computed "gc_refs" refcount. static int visit_decref(PyObject *op, void *arg) @@ -1006,6 +1023,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) _PyEval_StopTheWorld(interp); // merge refcounts for all queued objects merge_all_queued_objects(interp, state); + process_delayed_frees(interp); // Find unreachable objects int err = deduce_unreachable_heap(interp, state); @@ -1758,16 +1776,4 @@ _PyGC_ClearAllFreeLists(PyInterpreterState *interp) HEAD_UNLOCK(&_PyRuntime); } -void -_PyGC_Clear_DelayedObjects(PyInterpreterState *interp) -{ - HEAD_LOCK(&_PyRuntime); - PyThreadState *tstate = interp->threads.head; - while (tstate != NULL) { - _PyMem_ProcessDelayed(tstate); - tstate = (PyThreadState *)tstate->next; - } - HEAD_UNLOCK(&_PyRuntime); -} - #endif // Py_GIL_DISABLED From a187b03cf0f5aa019529f32241031805186ceb60 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Sun, 3 Mar 2024 02:32:38 +0900 Subject: [PATCH 5/5] nit --- Lib/test/support/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index aa6f71d0aad3cc..14e3766a34377b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -786,7 +786,6 @@ def gc_collect(): gc.collect() gc.collect() - @contextlib.contextmanager def disable_gc(): import gc