Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1d976b2

Browse files
markshannonmarkshannon
authored
GH-106485: Handle dict subclasses correctly when dematerializing __dict__ (GH-107837)
1 parent bafedfb commit 1d976b2

7 files changed

Lines changed: 169 additions & 7 deletions

File tree

‎Include/internal/pycore_dict.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
199199
}
200200

201201
extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
202-
extern int _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
202+
extern bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
203203
extern PyObject *_PyDict_FromItems(
204204
PyObject *const *keys, Py_ssize_t keys_offset,
205205
PyObject *const *values, Py_ssize_t values_offset,

‎Include/internal/pycore_global_objects_fini_generated.h‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_global_strings.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ struct _Py_global_strings {
3333
STRUCT_FOR_STR(anon_lambda, "<lambda>")
3434
STRUCT_FOR_STR(anon_listcomp, "<listcomp>")
3535
STRUCT_FOR_STR(anon_module, "<module>")
36+
STRUCT_FOR_STR(anon_null, "<NULL>")
3637
STRUCT_FOR_STR(anon_setcomp, "<setcomp>")
3738
STRUCT_FOR_STR(anon_string, "<string>")
3839
STRUCT_FOR_STR(anon_unknown, "<unknown>")

‎Include/internal/pycore_runtime_init_generated.h‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/test/test_opcache.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1+
import copy
2+
import pickle
13
import dis
24
import threading
35
import types
46
import unittest
57
from test.support import threading_helper
8+
import _testinternalcapi
69

710

811
class TestLoadSuperAttrCache(unittest.TestCase):
@@ -997,6 +1000,124 @@ def write(items):
9971000
opname = "UNPACK_SEQUENCE_LIST"
9981001
self.assert_races_do_not_crash(opname, get_items, read, write)
9991002

1003+
class C:
1004+
pass
1005+
1006+
class TestInstanceDict(unittest.TestCase):
1007+
1008+
def setUp(self):
1009+
c = C()
1010+
c.a, c.b, c.c = 0,0,0
1011+
1012+
def test_values_on_instance(self):
1013+
c = C()
1014+
c.a = 1
1015+
C().b = 2
1016+
c.c = 3
1017+
self.assertEqual(
1018+
_testinternalcapi.get_object_dict_values(c),
1019+
(1, '<NULL>', 3)
1020+
)
1021+
1022+
def test_dict_materialization(self):
1023+
c = C()
1024+
c.a = 1
1025+
c.b = 2
1026+
c.__dict__
1027+
self.assertIs(
1028+
_testinternalcapi.get_object_dict_values(c),
1029+
None
1030+
)
1031+
1032+
def test_dict_dematerialization(self):
1033+
c = C()
1034+
c.a = 1
1035+
c.b = 2
1036+
c.__dict__
1037+
self.assertIs(
1038+
_testinternalcapi.get_object_dict_values(c),
1039+
None
1040+
)
1041+
for _ in range(100):
1042+
c.a
1043+
self.assertEqual(
1044+
_testinternalcapi.get_object_dict_values(c),
1045+
(1, 2, '<NULL>')
1046+
)
1047+
1048+
def test_dict_dematerialization_multiple_refs(self):
1049+
c = C()
1050+
c.a = 1
1051+
c.b = 2
1052+
d = c.__dict__
1053+
for _ in range(100):
1054+
c.a
1055+
self.assertIs(
1056+
_testinternalcapi.get_object_dict_values(c),
1057+
None
1058+
)
1059+
self.assertIs(c.__dict__, d)
1060+
1061+
def test_dict_dematerialization_copy(self):
1062+
c = C()
1063+
c.a = 1
1064+
c.b = 2
1065+
c2 = copy.copy(c)
1066+
for _ in range(100):
1067+
c.a
1068+
c2.a
1069+
self.assertEqual(
1070+
_testinternalcapi.get_object_dict_values(c),
1071+
(1, 2, '<NULL>')
1072+
)
1073+
self.assertEqual(
1074+
_testinternalcapi.get_object_dict_values(c2),
1075+
(1, 2, '<NULL>')
1076+
)
1077+
c3 = copy.deepcopy(c)
1078+
for _ in range(100):
1079+
c.a
1080+
c3.a
1081+
self.assertEqual(
1082+
_testinternalcapi.get_object_dict_values(c),
1083+
(1, 2, '<NULL>')
1084+
)
1085+
#NOTE -- c3.__dict__ does not de-materialize
1086+
1087+
def test_dict_dematerialization_pickle(self):
1088+
c = C()
1089+
c.a = 1
1090+
c.b = 2
1091+
c2 = pickle.loads(pickle.dumps(c))
1092+
for _ in range(100):
1093+
c.a
1094+
c2.a
1095+
self.assertEqual(
1096+
_testinternalcapi.get_object_dict_values(c),
1097+
(1, 2, '<NULL>')
1098+
)
1099+
self.assertEqual(
1100+
_testinternalcapi.get_object_dict_values(c2),
1101+
(1, 2, '<NULL>')
1102+
)
1103+
1104+
def test_dict_dematerialization_subclass(self):
1105+
class D(dict): pass
1106+
c = C()
1107+
c.a = 1
1108+
c.b = 2
1109+
c.__dict__ = D(c.__dict__)
1110+
for _ in range(100):
1111+
c.a
1112+
self.assertIs(
1113+
_testinternalcapi.get_object_dict_values(c),
1114+
None
1115+
)
1116+
self.assertEqual(
1117+
c.__dict__,
1118+
{'a':1, 'b':2}
1119+
)
1120+
10001121

10011122
if __name__ == "__main__":
10021123
unittest.main()

‎Modules/_testinternalcapi.c‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "pycore_bytesobject.h" // _PyBytes_Find()
1616
#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc
1717
#include "pycore_ceval.h" // _PyEval_AddPendingCall
18+
#include "pycore_dict.h" // _PyDictOrValues_GetValues
1819
#include "pycore_fileutils.h" // _Py_normpath
1920
#include "pycore_frame.h" // _PyInterpreterFrame
2021
#include "pycore_gc.h" // PyGC_Head
@@ -1530,6 +1531,40 @@ test_pymem_getallocatorsname(PyObject *self, PyObject *args)
15301531
return PyUnicode_FromString(name);
15311532
}
15321533

1534+
static PyObject *
1535+
get_object_dict_values(PyObject *self, PyObject *obj)
1536+
{
1537+
PyTypeObject *type = Py_TYPE(obj);
1538+
if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) {
1539+
Py_RETURN_NONE;
1540+
}
1541+
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
1542+
if (!_PyDictOrValues_IsValues(dorv)) {
1543+
Py_RETURN_NONE;
1544+
}
1545+
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
1546+
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
1547+
assert(keys != NULL);
1548+
int size = (int)keys->dk_nentries;
1549+
assert(size >= 0);
1550+
PyObject *res = PyTuple_New(size);
1551+
if (res == NULL) {
1552+
return NULL;
1553+
}
1554+
_Py_DECLARE_STR(anon_null, "<NULL>");
1555+
for(int i = 0; i < size; i++) {
1556+
PyObject *item = values->values[i];
1557+
if (item == NULL) {
1558+
item = &_Py_STR(anon_null);
1559+
}
1560+
else {
1561+
Py_INCREF(item);
1562+
}
1563+
PyTuple_SET_ITEM(res, i, item);
1564+
}
1565+
return res;
1566+
}
1567+
15331568

15341569
static PyMethodDef module_functions[] = {
15351570
{"get_configs", get_configs, METH_NOARGS},
@@ -1594,6 +1629,7 @@ static PyMethodDef module_functions[] = {
15941629
{"check_pyobject_uninitialized_is_freed",
15951630
check_pyobject_uninitialized_is_freed, METH_NOARGS},
15961631
{"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS},
1632+
{"get_object_dict_values", get_object_dict_values, METH_O},
15971633
{NULL, NULL} /* sentinel */
15981634
};
15991635

‎Objects/dictobject.c‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5464,22 +5464,24 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values)
54645464
return make_dict_from_instance_attributes(interp, keys, values);
54655465
}
54665466

5467-
// Return 1 if the dict was dematerialized, 0 otherwise.
5468-
int
5467+
// Return true if the dict was dematerialized, false otherwise.
5468+
bool
54695469
_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv)
54705470
{
54715471
assert(_PyObject_DictOrValuesPointer(obj) == dorv);
54725472
assert(!_PyDictOrValues_IsValues(*dorv));
54735473
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv);
54745474
if (dict == NULL) {
5475-
return 0;
5475+
return false;
54765476
}
54775477
// It's likely that this dict still shares its keys (if it was materialized
54785478
// on request and not heavily modified):
5479-
assert(PyDict_CheckExact(dict));
5479+
if (!PyDict_CheckExact(dict)) {
5480+
return false;
5481+
}
54805482
assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE));
54815483
if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) {
5482-
return 0;
5484+
return false;
54835485
}
54845486
assert(dict->ma_values);
54855487
// We have an opportunity to do something *really* cool: dematerialize it!
@@ -5490,7 +5492,7 @@ _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv)
54905492
dict->ma_keys = NULL;
54915493
dict->ma_values = NULL;
54925494
Py_DECREF(dict);
5493-
return 1;
5495+
return true;
54945496
}
54955497

54965498
int

0 commit comments

Comments
 (0)