Thanks to visit codestin.com
Credit goes to github.com

Skip to content

gh-99741: Fix the Cross-Interpreter Data API #99939

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 2, 2022
Merged

gh-99741: Fix the Cross-Interpreter Data API #99939

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b8a9dba
Use a weakref in _PyCrossInterpreterData_RegisterClass().
ericsnowcurrently Nov 29, 2022
20c3a6a
Add _PyCrossInterpreterData_UnregisterClass().
ericsnowcurrently Nov 30, 2022
ce4e158
Clear cross-interpreter data after releasing it.
ericsnowcurrently Dec 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Include/cpython/pystate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,4 +403,5 @@ PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *);
typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *);

PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc);
PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *);
PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *);
5 changes: 3 additions & 2 deletions Include/internal/pycore_interp.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,10 @@ extern void _PyInterpreterState_Clear(PyThreadState *tstate);
struct _xidregitem;

struct _xidregitem {
PyTypeObject *cls;
crossinterpdatafunc getdata;
struct _xidregitem *prev;
struct _xidregitem *next;
PyObject *cls; // weakref to a PyTypeObject
crossinterpdatafunc getdata;
};

PyAPI_FUNC(PyInterpreterState*) _PyInterpreterState_LookUpID(int64_t);
Expand Down
107 changes: 84 additions & 23 deletions Python/pystate.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1873,8 +1873,9 @@ _release_xidata(void *arg)
_PyCrossInterpreterData *data = (_PyCrossInterpreterData *)arg;
if (data->free != NULL) {
data->free(data->data);
data->data = NULL;
}
Py_XDECREF(data->obj);
Py_CLEAR(data->obj);
}

static void
Expand All @@ -1894,6 +1895,8 @@ _call_in_interpreter(struct _gilstate_runtime_state *gilstate,
save_tstate = _PyThreadState_Swap(gilstate, tstate);
}

// XXX Once the GIL is per-interpreter, this should be called with the
// calling interpreter's GIL released and the target interpreter's held.
func(arg);

// Switch back.
Expand Down Expand Up @@ -1938,21 +1941,73 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data)
crossinterpdatafunc. It would be simpler and more efficient. */

static int
_register_xidata(struct _xidregistry *xidregistry, PyTypeObject *cls,
_xidregistry_add_type(struct _xidregistry *xidregistry, PyTypeObject *cls,
crossinterpdatafunc getdata)
{
// Note that we effectively replace already registered classes
// rather than failing.
struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem));
if (newhead == NULL)
if (newhead == NULL) {
return -1;
newhead->cls = cls;
}
// XXX Assign a callback to clear the entry from the registry?
newhead->cls = PyWeakref_NewRef((PyObject *)cls, NULL);
if (newhead->cls == NULL) {
PyMem_RawFree(newhead);
return -1;
}
newhead->getdata = getdata;
newhead->prev = NULL;
newhead->next = xidregistry->head;
if (newhead->next != NULL) {
newhead->next->prev = newhead;
}
xidregistry->head = newhead;
return 0;
}

static struct _xidregitem *
_xidregistry_remove_entry(struct _xidregistry *xidregistry,
struct _xidregitem *entry)
{
struct _xidregitem *next = entry->next;
if (entry->prev != NULL) {
assert(entry->prev->next == entry);
entry->prev->next = next;
}
else {
assert(xidregistry->head == entry);
xidregistry->head = next;
}
if (next != NULL) {
next->prev = entry->prev;
}
Py_DECREF(entry->cls);
PyMem_RawFree(entry);
return next;
}

static struct _xidregitem *
_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls)
{
struct _xidregitem *cur = xidregistry->head;
while (cur != NULL) {
PyObject *registered = PyWeakref_GetObject(cur->cls);
if (registered == Py_None) {
// The weakly ref'ed object was freed.
cur = _xidregistry_remove_entry(xidregistry, cur);
}
else {
assert(PyType_Check(registered));
if (registered == (PyObject *)cls) {
return cur;
}
cur = cur->next;
}
}
return NULL;
}

static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry);

int
Expand All @@ -1968,19 +2023,32 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls,
return -1;
}

// Make sure the class isn't ever deallocated.
Py_INCREF((PyObject *)cls);

struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ;
PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK);
if (xidregistry->head == NULL) {
_register_builtins_for_crossinterpreter_data(xidregistry);
}
int res = _register_xidata(xidregistry, cls, getdata);
int res = _xidregistry_add_type(xidregistry, cls, getdata);
PyThread_release_lock(xidregistry->mutex);
return res;
}

int
_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls)
{
int res = 0;
struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ;
PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK);
struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls);
if (matched != NULL) {
(void)_xidregistry_remove_entry(xidregistry, matched);
res = 1;
}
PyThread_release_lock(xidregistry->mutex);
return res;
}


/* Cross-interpreter objects are looked up by exact match on the class.
We can reassess this policy when we move from a global registry to a
tp_* slot. */
Expand All @@ -1990,22 +2058,15 @@ _PyCrossInterpreterData_Lookup(PyObject *obj)
{
struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ;
PyObject *cls = PyObject_Type(obj);
crossinterpdatafunc getdata = NULL;
PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK);
struct _xidregitem *cur = xidregistry->head;
if (cur == NULL) {
if (xidregistry->head == NULL) {
_register_builtins_for_crossinterpreter_data(xidregistry);
cur = xidregistry->head;
}
for(; cur != NULL; cur = cur->next) {
if (cur->cls == (PyTypeObject *)cls) {
getdata = cur->getdata;
break;
}
}
struct _xidregitem *matched = _xidregistry_find_type(xidregistry,
(PyTypeObject *)cls);
Py_DECREF(cls);
PyThread_release_lock(xidregistry->mutex);
return getdata;
return matched != NULL ? matched->getdata : NULL;
}

/* cross-interpreter data for builtin types */
Expand Down Expand Up @@ -2111,22 +2172,22 @@ static void
_register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry)
{
// None
if (_register_xidata(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) {
if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) {
Py_FatalError("could not register None for cross-interpreter sharing");
}

// int
if (_register_xidata(xidregistry, &PyLong_Type, _long_shared) != 0) {
if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) {
Py_FatalError("could not register int for cross-interpreter sharing");
}

// bytes
if (_register_xidata(xidregistry, &PyBytes_Type, _bytes_shared) != 0) {
if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) {
Py_FatalError("could not register bytes for cross-interpreter sharing");
}

// str
if (_register_xidata(xidregistry, &PyUnicode_Type, _str_shared) != 0) {
if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) {
Py_FatalError("could not register str for cross-interpreter sharing");
}
}
Expand Down