From 6c2ffb8459fc9092c639b332d033a4267885e5ef Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 1 Nov 2023 00:09:30 +0900 Subject: [PATCH 1/4] gh-110481: Implement _Py_DECREF_NO_DEALLOC for free-threaded build --- Include/internal/pycore_object.h | 49 +++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 206d8a5d4cc5e1..31445bd31bcc85 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -210,6 +210,10 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) } #else +// Merge the local and shared reference count fields and add `extra` to the +// refcount when merging. +Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); + // TODO: implement Py_DECREF specializations for Py_NOGIL build static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) @@ -220,7 +224,47 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) static inline void _Py_DECREF_NO_DEALLOC(PyObject *op) { - Py_DECREF(op); + if (_Py_IsImmortal(op)) { + return; + } + + if (_Py_IsOwnedByCurrentThread(op)) { + uint32_t refcount = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + assert(refcount != 0); + refcount--; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, refcount); +#ifdef Py_REF_DEBUG + if (refcount <= 0) { + _Py_FatalRefcountError("Expected a positive remaining refcount"); + } +#endif + } + else { + Py_ssize_t refcount = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + assert(refcount != 0); + Py_ssize_t new_shared; + int should_queue = refcount == _Py_REF_MAYBE_WEAKREF; + do { + if (should_queue) { + new_shared = _Py_REF_QUEUED; + } + else { + new_shared = refcount - (1 << _Py_REF_SHARED_SHIFT); + } + } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, + &refcount, new_shared)); + + if (should_queue) { + // TODO: the inter-thread queue is not yet implemented. For now, + // we just merge the refcount here. + refcount = _Py_ExplicitMergeRefcount(op, -1); +#ifdef Py_REF_DEBUG + if (refcount <= 0) { + _Py_FatalRefcountError("Expected a positive remaining refcount"); + } +#endif + } + } } static inline int @@ -235,9 +279,6 @@ _Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared) return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED; } -// Merge the local and shared reference count fields and add `extra` to the -// refcount when merging. -Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); #endif // !defined(Py_NOGIL) #ifdef Py_REF_DEBUG From 397f0189cb7549bb5b1bf3300ac28909e6a7d332 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 1 Nov 2023 01:47:41 +0900 Subject: [PATCH 2/4] Remove assert --- Include/internal/pycore_object.h | 1 - 1 file changed, 1 deletion(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 31445bd31bcc85..27ddfcfd66088f 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -241,7 +241,6 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) } else { Py_ssize_t refcount = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); - assert(refcount != 0); Py_ssize_t new_shared; int should_queue = refcount == _Py_REF_MAYBE_WEAKREF; do { From 2c2f4df0d282355ed555209fd3f310179ae12083 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 1 Nov 2023 01:49:21 +0900 Subject: [PATCH 3/4] Update --- Include/internal/pycore_object.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 27ddfcfd66088f..69287670da583f 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -242,7 +242,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) else { Py_ssize_t refcount = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); Py_ssize_t new_shared; - int should_queue = refcount == _Py_REF_MAYBE_WEAKREF; + int should_queue = (refcount == 0 || refcount == _Py_REF_MAYBE_WEAKREF); do { if (should_queue) { new_shared = _Py_REF_QUEUED; From 8166ae097a6950c5ce9117fb3eca9ac7b533dccb Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 1 Nov 2023 02:10:26 +0900 Subject: [PATCH 4/4] Update --- Include/internal/pycore_object.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 69287670da583f..035891015b50d2 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -233,15 +233,15 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) assert(refcount != 0); refcount--; _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, refcount); -#ifdef Py_REF_DEBUG - if (refcount <= 0) { - _Py_FatalRefcountError("Expected a positive remaining refcount"); + if (refcount == 0) { + // Assume that local + share >= 1 + _Py_MergeZeroLocalRefcount(op); } -#endif } else { Py_ssize_t refcount = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); Py_ssize_t new_shared; + // Shared refcount can be zero but we should consider local refcount. int should_queue = (refcount == 0 || refcount == _Py_REF_MAYBE_WEAKREF); do { if (should_queue) {