Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit a703a21

Browse files
gpsheadgpshead
committed
* Use weakref's of DBCursor objects for the iterator cursors to avoid a
 memory leak that would've occurred for all iterators that were destroyed before having iterated until they raised StopIteration. * Simplify some code. * Add new test cases to check for the memleak and ensure that mixing iteration with modification of the values for existing keys works.
1 parent 83c1874 commit a703a21

3 files changed

Lines changed: 159 additions & 52 deletions

File tree

‎Lib/bsddb/__init__.py‎

Lines changed: 45 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -67,77 +67,76 @@
6767
if sys.version >= '2.3':
6868
exec """
6969
import UserDict
70+
from weakref import ref
7071
class _iter_mixin(UserDict.DictMixin):
72+
def _make_iter_cursor(self):
73+
cur = self.db.cursor()
74+
key = id(cur)
75+
self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))
76+
return cur
77+
78+
def _gen_cref_cleaner(self, key):
79+
# use generate the function for the weakref callback here
80+
# to ensure that we do not hold a strict reference to cur
81+
# in the callback.
82+
return lambda ref: self._cursor_refs.pop(key, None)
83+
7184
def __iter__(self):
7285
try:
73-
cur = self.db.cursor()
74-
self._iter_cursors[str(cur)] = cur
86+
cur = self._make_iter_cursor()
87+
88+
# FIXME-20031102-greg: race condition. cursor could
89+
# be closed by another thread before this call.
7590
7691
# since we're only returning keys, we call the cursor
7792
# methods with flags=0, dlen=0, dofs=0
78-
curkey = cur.first(0,0,0)[0]
79-
yield curkey
93+
key = cur.first(0,0,0)[0]
94+
yield key
8095
8196
next = cur.next
8297
while 1:
8398
try:
84-
curkey = next(0,0,0)[0]
85-
yield curkey
99+
key = next(0,0,0)[0]
100+
yield key
86101
except _bsddb.DBCursorClosedError:
87-
# our cursor object was closed since we last yielded
88-
# create a new one and attempt to reposition to the
89-
# right place
90-
cur = self.db.cursor()
91-
self._iter_cursors[str(cur)] = cur
102+
cur = self._make_iter_cursor()
92103
# FIXME-20031101-greg: race condition. cursor could
93-
# be closed by another thread before this set call.
94-
try:
95-
cur.set(curkey,0,0,0)
96-
except _bsddb.DBCursorClosedError:
97-
# halt iteration on race condition...
98-
raise _bsddb.DBNotFoundError
104+
# be closed by another thread before this call.
105+
cur.set(key,0,0,0)
99106
next = cur.next
100107
except _bsddb.DBNotFoundError:
101-
try:
102-
del self._iter_cursors[str(cur)]
103-
except KeyError:
104-
pass
108+
return
109+
except _bsddb.DBCursorClosedError:
110+
# the database was modified during iteration. abort.
105111
return
106112
107113
def iteritems(self):
108114
try:
109-
cur = self.db.cursor()
110-
self._iter_cursors[str(cur)] = cur
115+
cur = self._make_iter_cursor()
116+
117+
# FIXME-20031102-greg: race condition. cursor could
118+
# be closed by another thread before this call.
111119
112120
kv = cur.first()
113-
curkey = kv[0]
121+
key = kv[0]
114122
yield kv
115123
116124
next = cur.next
117125
while 1:
118126
try:
119127
kv = next()
120-
curkey = kv[0]
128+
key = kv[0]
121129
yield kv
122130
except _bsddb.DBCursorClosedError:
123-
# our cursor object was closed since we last yielded
124-
# create a new one and attempt to reposition to the
125-
# right place
126-
cur = self.db.cursor()
127-
self._iter_cursors[str(cur)] = cur
131+
cur = self._make_iter_cursor()
128132
# FIXME-20031101-greg: race condition. cursor could
129-
# be closed by another thread before this set call.
130-
try:
131-
cur.set(curkey,0,0,0)
132-
except _bsddb.DBCursorClosedError:
133-
# halt iteration on race condition...
134-
raise _bsddb.DBNotFoundError
133+
# be closed by another thread before this call.
134+
cur.set(key,0,0,0)
135135
next = cur.next
136136
except _bsddb.DBNotFoundError:
137-
try:
138-
del self._iter_cursors[str(cur)]
139-
except KeyError:
140-
pass
137+
return
138+
except _bsddb.DBCursorClosedError:
139+
# the database was modified during iteration. abort.
141140
return
142141
"""
143142
else:
@@ -159,7 +158,7 @@ def __init__(self, db):
159158
# thread while doing a put or delete in another thread. The
160159
# reason is that _checkCursor and _closeCursors are not atomic
161160
# operations. Doing our own locking around self.dbc,
162-
# self.saved_dbc_key and self._iter_cursors could prevent this.
161+
# self.saved_dbc_key and self._cursor_refs could prevent this.
163162
# TODO: A test case demonstrating the problem needs to be written.
164163

165164
# self.dbc is a DBCursor object used to implement the
@@ -169,15 +168,11 @@ def __init__(self, db):
169168

170169
# a collection of all DBCursor objects currently allocated
171170
# by the _iter_mixin interface.
172-
self._iter_cursors = {}
173-
171+
self._cursor_refs = {}
174172

175173
def __del__(self):
176174
self.close()
177175

178-
def _get_dbc(self):
179-
return self.dbc
180-
181176
def _checkCursor(self):
182177
if self.dbc is None:
183178
self.dbc = self.db.cursor()
@@ -197,7 +192,10 @@ def _closeCursors(self, save=True):
197192
self.saved_dbc_key = c.current(0,0,0)[0]
198193
c.close()
199194
del c
200-
map(lambda c: c.close(), self._iter_cursors.values())
195+
for cref in self._cursor_refs.values():
196+
c = cref()
197+
if c is not None:
198+
c.close()
201199

202200
def _checkOpen(self):
203201
if self.db is None:

‎Lib/test/test_bsddb.py‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
Adapted to unittest format and expanded scope by Raymond Hettinger
44
"""
55
import os, sys
6+
import copy
67
import bsddb
78
import dbhash # Just so we know it's imported
89
import unittest
@@ -64,6 +65,56 @@ def test_mapping_iteration_methods(self):
6465
self.assertSetEquals(d.itervalues(), f.itervalues())
6566
self.assertSetEquals(d.iteritems(), f.iteritems())
6667

68+
def test_iter_while_modifying_values(self):
69+
if not hasattr(self.f, '__iter__'):
70+
return
71+
72+
di = iter(self.d)
73+
while 1:
74+
try:
75+
key = di.next()
76+
self.d[key] = 'modified '+key
77+
except StopIteration:
78+
break
79+
80+
# it should behave the same as a dict. modifying values
81+
# of existing keys should not break iteration. (adding
82+
# or removing keys should)
83+
fi = iter(self.f)
84+
while 1:
85+
try:
86+
key = fi.next()
87+
self.f[key] = 'modified '+key
88+
except StopIteration:
89+
break
90+
91+
self.test_mapping_iteration_methods()
92+
93+
def test_iteritems_while_modifying_values(self):
94+
if not hasattr(self.f, 'iteritems'):
95+
return
96+
97+
di = self.d.iteritems()
98+
while 1:
99+
try:
100+
k, v = di.next()
101+
self.d[k] = 'modified '+v
102+
except StopIteration:
103+
break
104+
105+
# it should behave the same as a dict. modifying values
106+
# of existing keys should not break iteration. (adding
107+
# or removing keys should)
108+
fi = self.f.iteritems()
109+
while 1:
110+
try:
111+
k, v = fi.next()
112+
self.f[k] = 'modified '+v
113+
except StopIteration:
114+
break
115+
116+
self.test_mapping_iteration_methods()
117+
67118
def test_first_next_looping(self):
68119
items = [self.f.first()]
69120
for i in xrange(1, len(self.f)):
@@ -111,15 +162,16 @@ def test__no_deadlock_first(self, debug=0):
111162
# the cursor's read lock will deadlock the write lock request..
112163

113164
# test the iterator interface (if present)
114-
if hasattr(self, 'iteritems'):
165+
if hasattr(self.f, 'iteritems'):
115166
if debug: print "D"
116-
k,v = self.f.iteritems()
167+
i = self.f.iteritems()
168+
k,v = i.next()
117169
if debug: print "E"
118170
self.f[k] = "please don't deadlock"
119171
if debug: print "F"
120172
while 1:
121173
try:
122-
k,v = self.f.iteritems()
174+
k,v = i.next()
123175
except StopIteration:
124176
break
125177
if debug: print "F2"
@@ -144,6 +196,27 @@ def test__no_deadlock_first(self, debug=0):
144196
self.f[k] = "be gone with ye deadlocks"
145197
self.assert_(self.f[k], "be gone with ye deadlocks")
146198

199+
def test_for_cursor_memleak(self):
200+
if not hasattr(self.f, 'iteritems'):
201+
return
202+
203+
# do the bsddb._DBWithCursor _iter_mixin internals leak cursors?
204+
nc1 = len(self.f._cursor_refs)
205+
# create iterator
206+
i = self.f.iteritems()
207+
nc2 = len(self.f._cursor_refs)
208+
# use the iterator (should run to the first yeild, creating the cursor)
209+
k, v = i.next()
210+
nc3 = len(self.f._cursor_refs)
211+
# destroy the iterator; this should cause the weakref callback
212+
# to remove the cursor object from self.f._cursor_refs
213+
del i
214+
nc4 = len(self.f._cursor_refs)
215+
216+
self.assertEqual(nc1, nc2)
217+
self.assertEqual(nc1, nc4)
218+
self.assert_(nc3 == nc1+1)
219+
147220
def test_popitem(self):
148221
k, v = self.f.popitem()
149222
self.assert_(k in self.d)

‎Modules/_bsddb.c‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484

8585
/* --------------------------------------------------------------------- */
8686

87+
#include <stddef.h> /* for offsetof() */
8788
#include <Python.h>
8889
#include <db.h>
8990

@@ -92,8 +93,11 @@
9293

9394
/* 40 = 4.0, 33 = 3.3; this will break if the second number is > 9 */
9495
#define DBVER (DB_VERSION_MAJOR * 10 + DB_VERSION_MINOR)
96+
#if DB_VERSION_MINOR > 9
97+
#error "eek! DBVER can't handle minor versions > 9"
98+
#endif
9599

96-
#define PY_BSDDB_VERSION "4.2.3"
100+
#define PY_BSDDB_VERSION "4.2.4"
97101
static char *rcs_id = "$Id$";
98102

99103

@@ -184,6 +188,12 @@ static PyObject* DBPermissionsError; /* EPERM */
184188
/* --------------------------------------------------------------------- */
185189
/* Structure definitions */
186190

191+
#if PYTHON_API_VERSION >= 1010 /* python >= 2.1 support weak references */
192+
#define HAVE_WEAKREF
193+
#else
194+
#undef HAVE_WEAKREF
195+
#endif
196+
187197
struct behaviourFlags {
188198
/* What is the default behaviour when DB->get or DBCursor->get returns a
189199
DB_NOTFOUND error? Return None or raise an exception? */
@@ -194,7 +204,7 @@ struct behaviourFlags {
194204
};
195205

196206
#define DEFAULT_GET_RETURNS_NONE 1
197-
#define DEFAULT_CURSOR_SET_RETURNS_NONE 0 /* 0 in pybsddb < 4.2, python < 2.4 */
207+
#define DEFAULT_CURSOR_SET_RETURNS_NONE 1 /* 0 in pybsddb < 4.2, python < 2.4 */
198208

199209
typedef struct {
200210
PyObject_HEAD
@@ -224,6 +234,9 @@ typedef struct {
224234
PyObject_HEAD
225235
DBC* dbc;
226236
DBObject* mydb;
237+
#ifdef HAVE_WEAKREF
238+
PyObject *in_weakreflist; /* List of weak references */
239+
#endif
227240
} DBCursorObject;
228241

229242

@@ -760,6 +773,9 @@ newDBCursorObject(DBC* dbc, DBObject* db)
760773

761774
self->dbc = dbc;
762775
self->mydb = db;
776+
#ifdef HAVE_WEAKREF
777+
self->in_weakreflist = NULL;
778+
#endif
763779
Py_INCREF(self->mydb);
764780
return self;
765781
}
@@ -769,6 +785,13 @@ static void
769785
DBCursor_dealloc(DBCursorObject* self)
770786
{
771787
int err;
788+
789+
#ifdef HAVE_WEAKREF
790+
if (self->in_weakreflist != NULL) {
791+
PyObject_ClearWeakRefs((PyObject *) self);
792+
}
793+
#endif
794+
772795
if (self->dbc != NULL) {
773796
MYDB_BEGIN_ALLOW_THREADS;
774797
/* If the underlying database has been closed, we don't
@@ -4252,6 +4275,19 @@ statichere PyTypeObject DBCursor_Type = {
42524275
0, /*tp_as_sequence*/
42534276
0, /*tp_as_mapping*/
42544277
0, /*tp_hash*/
4278+
#ifdef HAVE_WEAKREF
4279+
0, /* tp_call */
4280+
0, /* tp_str */
4281+
0, /* tp_getattro */
4282+
0, /* tp_setattro */
4283+
0, /* tp_as_buffer */
4284+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_WEAKREFS, /* tp_flags */
4285+
0, /* tp_doc */
4286+
0, /* tp_traverse */
4287+
0, /* tp_clear */
4288+
0, /* tp_richcompare */
4289+
offsetof(DBCursorObject, in_weakreflist), /* tp_weaklistoffset */
4290+
#endif
42554291
};
42564292

42574293

0 commit comments

Comments
 (0)