gh-112075: Support freeing object memory via QSBR (#116344)

DinoV · web-flow · commit 7db871e4fa48 · 2024-03-08T09:56:36.000-08:00
Free objects with qsbr if shared
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
@@ -44,6 +44,7 @@ static inline PyObject* _Py_FROM_GC(PyGC_Head *gc) {
 #  define _PyGC_BITS_UNREACHABLE    (4)
 #  define _PyGC_BITS_FROZEN         (8)
 #  define _PyGC_BITS_SHARED         (16)
+#  define _PyGC_BITS_SHARED_INLINE  (32)
 #endif
 
 /* True if the object is currently tracked by the GC. */
@@ -71,9 +72,12 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) {
 
 #ifdef Py_GIL_DISABLED
 
-/* True if an object is shared between multiple threads and
- * needs special purpose when freeing to do the possibility
- * of in-flight lock-free reads occurring */
+/* True if memory the object references is shared between
+ * multiple threads and needs special purpose when freeing
+ * those references due to the possibility of in-flight
+ * lock-free reads occurring.  The object is responsible
+ * for calling _PyMem_FreeDelayed on the referenced
+ * memory. */
 static inline int _PyObject_GC_IS_SHARED(PyObject *op) {
     return (op->ob_gc_bits & _PyGC_BITS_SHARED) != 0;
 }
@@ -84,6 +88,23 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
 }
 #define _PyObject_GC_SET_SHARED(op) _PyObject_GC_SET_SHARED(_Py_CAST(PyObject*, op))
 
+/* True if the memory of the object is shared between multiple
+ * threads and needs special purpose when freeing due to
+ * the possibility of in-flight lock-free reads occurring.
+ * Objects with this bit that are GC objects will automatically
+ * delay-freed by PyObject_GC_Del.  */
+static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) {
+    return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0;
+}
+#define _PyObject_GC_IS_SHARED_INLINE(op) \
+    _PyObject_GC_IS_SHARED_INLINE(_Py_CAST(PyObject*, op))
+
+static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) {
+    op->ob_gc_bits |= _PyGC_BITS_SHARED_INLINE;
+}
+#define _PyObject_GC_SET_SHARED_INLINE(op) \
+    _PyObject_GC_SET_SHARED_INLINE(_Py_CAST(PyObject*, op))
+
 #endif
 
 /* Bit flags for _gc_prev */
diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h
@@ -119,6 +119,9 @@ extern int _PyMem_DebugEnabled(void);
 // Enqueue a pointer to be freed possibly after some delay.
 extern void _PyMem_FreeDelayed(void *ptr);
 
+// Enqueue an object to be freed possibly after some delay
+extern void _PyObject_FreeDelayed(void *ptr);
+
 // Periodically process delayed free requests.
 extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
 
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
@@ -1070,7 +1070,7 @@ _PyMem_Strdup(const char *str)
 
 // A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
 struct _mem_work_item {
-    void *ptr;
+    uintptr_t ptr; // lowest bit tagged 1 for objects freed with PyObject_Free
     uint64_t qsbr_goal;
 };
 
@@ -1084,16 +1084,27 @@ struct _mem_work_chunk {
     struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
 };
 
-void
-_PyMem_FreeDelayed(void *ptr)
+static void
+free_work_item(uintptr_t ptr)
+{
+    if (ptr & 0x01) {
+        PyObject_Free((char *)(ptr - 1));
+    }
+    else {
+        PyMem_Free((void *)ptr);
+    }
+}
+
+static void
+free_delayed(uintptr_t ptr)
 {
 #ifndef Py_GIL_DISABLED
-    PyMem_Free(ptr);
+    free_work_item(ptr);
 #else
     if (_PyRuntime.stoptheworld.world_stopped) {
         // Free immediately if the world is stopped, including during
         // interpreter shutdown.
-        PyMem_Free(ptr);
+        free_work_item(ptr);
         return;
     }
 
@@ -1120,7 +1131,7 @@ _PyMem_FreeDelayed(void *ptr)
     if (buf == NULL) {
         // failed to allocate a buffer, free immediately
         _PyEval_StopTheWorld(tstate->base.interp);
-        PyMem_Free(ptr);
+        free_work_item(ptr);
         _PyEval_StartTheWorld(tstate->base.interp);
         return;
     }
@@ -1137,6 +1148,20 @@ _PyMem_FreeDelayed(void *ptr)
 #endif
 }
 
+void
+_PyMem_FreeDelayed(void *ptr)
+{
+    assert(!((uintptr_t)ptr & 0x01));
+    free_delayed((uintptr_t)ptr);
+}
+
+void
+_PyObject_FreeDelayed(void *ptr)
+{
+    assert(!((uintptr_t)ptr & 0x01));
+    free_delayed(((uintptr_t)ptr)|0x01);
+}
+
 static struct _mem_work_chunk *
 work_queue_first(struct llist_node *head)
 {
@@ -1156,7 +1181,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr,
                 return;
             }
 
-            PyMem_Free(item->ptr);
+            free_work_item(item->ptr);
             buf->rd_idx++;
         }
 
@@ -1243,7 +1268,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
             // Free the remaining items immediately. There should be no other
             // threads accessing the memory at this point during shutdown.
             struct _mem_work_item *item = &buf->array[buf->rd_idx];
-            PyMem_Free(item->ptr);
+            free_work_item(item->ptr);
             buf->rd_idx++;
         }
 
diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
@@ -1695,6 +1695,7 @@ PyObject_GC_Del(void *op)
 {
     size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type);
     if (_PyObject_GC_IS_TRACKED(op)) {
+        _PyObject_GC_UNTRACK(op);
 #ifdef Py_DEBUG
         PyObject *exc = PyErr_GetRaisedException();
         if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0,
@@ -1707,8 +1708,13 @@ PyObject_GC_Del(void *op)
     }
 
     record_deallocation(_PyThreadState_GET());
-
-    PyObject_Free(((char *)op)-presize);
+    PyObject *self = (PyObject *)op;
+    if (_PyObject_GC_IS_SHARED_INLINE(self)) {
+        _PyObject_FreeDelayed(((char *)op)-presize);
+    }
+    else {
+        PyObject_Free(((char *)op)-presize);
+    }
 }
 
 int
