Description
Bug report
Bug description:
Hi,
I've been slowly building/debugging my dumb locking idea, and I think I've hit a bug in upstream python.
My understanding of how the GC detects when objects are in use by C, is that it counts the number of references to an object, and subtracts that from the reference count fields. This works, except for if C is holding onto an object with a reference count of 0. Which is the state of the object when passed to .tp_dealloc.
Various parts of gc_free_threading.c increment the reference counter whilst objects are on various queues. However, (due to the paragraph above,) even tho the world is stopped, this is not a safe operation if the reference count is 0.
Order of events:
- Object last reference is deleted. Reference count becomes 0.
- .tp_dealloc is called.
- .tp_dealloc goes to sleep (eg, on a lock) before freeing the object.
- The GC is called from another thread.
- The world is stopped and the GC runs.
- The GC sees that the object has no incoming references and:
-- Increments the reference count to 1
-- Pushes it onto a "to dealloc" list - The world is started
- The GC thread decrements the reference count back to 0, and calls .tp_dealloc a second time.
This may show up as objects in .tp_dealloc seeing the reference count of objects unexpectedly going to 1. (Which for example, breaks _PyObject_ResurrectStart)
I ended up writing a reproducer to prove it wasn't my code breaking things. (Sorry, it is a bit horrid, but it does trigger the issue reliably on my machine):
#include <Python.h>
typedef struct {
PyObject_HEAD;
int dealloced;
} pyAtest;
static int gc_sync = 0;
static PyMutex lock = {0};
static void
yield(void)
{
#ifdef MS_WINDOWS
SwitchToThread();
#elif defined(HAVE_SCHED_H)
sched_yield();
#endif
}
static void
Atest_dealloc(PyObject *op) {
// RC thread
int val;
pyAtest* self = (pyAtest*) op;
// Check we haven't already been deallocated.
assert(!self->dealloced);
self->dealloced=1;
// Wait for collector to be ready.
Py_BEGIN_ALLOW_THREADS;
do {
val = 1;
yield();
} while (!_Py_atomic_compare_exchange_int(&gc_sync, &val, 2));
Py_END_ALLOW_THREADS;
// Collector holds the lock. Wait for the collector to start the GC
while (PyThreadState_Get()->eval_breaker == 0) {
yield();
}
// Now go to sleep so the GC can run
PyMutex_Lock(&lock);
PyMutex_Unlock(&lock);
PyObject_GC_UnTrack(op);
PyObject_GC_Del(op);
}
static PyObject*
atest_gc(PyObject *self, PyObject *args)
{
// GC thread
int val;
// We pick up the lock first
PyMutex_Lock(&lock);
// Sync the RC thread
Py_BEGIN_ALLOW_THREADS;
_Py_atomic_store_int(&gc_sync, 1);
do {
val = 2;
yield();
} while (!_Py_atomic_compare_exchange_int(&gc_sync, &val, 0));
Py_END_ALLOW_THREADS;
// the stop_the_world inside the PyGC_Collect() will wait till the RC
// thread is on PyMutex_Lock() before doing the GC.
PyGC_Collect();
PyMutex_Unlock(&lock);
return Py_None;
}
// All the boring stuff
static PyObject*
Atest_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
return type->tp_alloc(type, 0);
}
static int
Atest_traverse(PyObject *self, visitproc visit, void *arg)
{
return 0;
}
static PyTypeObject AtestObjectType = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_name = "atest.Atest",
.tp_basicsize = sizeof(pyAtest),
.tp_itemsize = 0,
.tp_new = Atest_new,
.tp_dealloc = (destructor)Atest_dealloc,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
.tp_traverse = Atest_traverse,
.tp_doc = "a test",
};
static int
atest_module_exec(PyObject *m)
{
if (PyType_Ready(&AtestObjectType) < 0) {
return -1;
}
if (PyModule_AddObjectRef(m, "ATest", (PyObject *) &AtestObjectType) < 0) {
return -1;
}
return 0;
}
static PyModuleDef_Slot atest_slots[] = {
{Py_mod_exec, atest_module_exec},
{Py_mod_gil, Py_MOD_GIL_NOT_USED},
{0, NULL},
};
static PyMethodDef atest_methods[] = {
{"gc", atest_gc, METH_VARARGS, "issue gc"},
{NULL, NULL, 0, NULL},
};
static struct PyModuleDef atest_module = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = "atest",
.m_size = 0,
.m_slots = atest_slots,
.m_methods = atest_methods,
};
PyMODINIT_FUNC
PyInit_atest(void)
{
return PyModuleDef_Init(&atest_module);
}import atest
import threading
t = threading.Thread(target=atest.gc)
t.start()
# Create and destroy an ATest
atest.ATest()
t.join()$ ./python test.py
python: ./Modules/atest.c:27: Atest_dealloc: Assertion `!self->dealloced' failed.
Aborted (core dumped)CPython versions tested on:
CPython main branch
Operating systems tested on:
Linux